fix build: Add a few includes, suppress some werrors

Not sure how this slipped through the cracks, but I'm guessing the build
broke when we upgraded the cpp compilers (to clang 17 and gcc 13).

* Include <cstdint> in a few places
* Suppress -Wstringop-overflow in sparkd-gcc-release build
* Change const char * param to explicit std::string for proper type
  deduction in call to 'std::make_tuple'. For some reason, the gcc build
  was failing with an array-bounds werror (changing the arg fixed it).

.devcontainer/Dockerfile

@@ -1,122 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Note installations that take a while (more than 10s)
-
-RUN apt-get update
-RUN apt-get install -y ca-certificates
-RUN apt-get install -y ccache
-RUN apt-get install -y curl
-RUN apt-get install -y fish
-RUN apt-get install -y gdb
-# git-all: ~250s
-RUN apt-get install -y git-all
-RUN apt-get install -y gnupg
-# liblttng-ust-dev: ~30s
-RUN apt-get install -y liblttng-ust-dev
-RUN apt-get install -y libssh2-1-dev
-RUN apt-get install -y locales
-RUN apt-get install -y ninja-build
-# openssh-server: ~40s
-RUN apt-get install -y openssh-server
-RUN apt-get install -y pkg-config
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y sudo
-RUN apt-get install -y tzdata
-RUN apt-get install -y vim
-RUN apt-get install -y zip
-
-# Python: ~30s
-RUN apt-get install -y \
-   python3-pip \
-   python3-venv \
-   && python3 -m pip install --upgrade pip
-
-# GCC
-ARG gcc_major=13
-RUN \
-   add-apt-repository ppa:ubuntu-toolchain-r/test \
-   && apt-get update \
-   && apt-get install -y g++-$gcc_major
-
-# Clang
-ARG clang_major=17
-RUN \
-   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
-   | tee -a "/etc/apt/sources.list.d/llvm.list" \
-   && apt-get update \
-   && apt-get install -y \
-   clang-$clang_major \
-   clang-format-$clang_major \
-   clangd-$clang_major
-
-# CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
-
-# Dotnet
-RUN apt-get update && apt-get install -y dotnet7
-
-# Node
-ARG node_major=18
-RUN mkdir -p /etc/apt/keyrings \
-   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
-   | tee /etc/apt/sources.list.d/nodesource.list \
-   && apt-get update \
-   && apt-get install -y nodejs
-
-# Emumba (deps)
-RUN apt-get update && apt-get -y install rapidjson-dev
-# Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
-
-# Pcap
-RUN apt-get update && apt-get install -y libpcap0.8-dev
-
-# Set up default executables (update-alternatives)
-RUN \
-   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
-   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
-   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
-   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
-
-# Add compiler symlinks for ccache 
-RUN \
-   ln -s /usr/bin/ccache /usr/local/bin/clang; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-   groupadd -g "$docker_uid" "$docker_user" \
-   && useradd \
-   -m \
-   -u "$docker_uid" \
-   -g "$docker_uid" \
-   -G sudo \
-   --password "$(openssl passwd -1 "$docker_pass")" \
-   "$docker_user"

.devcontainer/devcontainer.json

@@ -1,48 +0,0 @@
-{
-  "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
-  "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
-  },
-  // "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
-  "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
-  "updateRemoteUserUID": true,
-  "shutdownAction": "none",
-  // Note: If running multiple containers on the same host, each must use a
-  // unique set of ports. Usually we just add 1000 to each port as needed. For
-  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
-  "runArgs": [
-    "--shm-size",
-    "2gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003"
-  ],
-  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
-  ],
-  "customizations": {
-    "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
-      "extensions": [
-        "eamodio.gitlens",
-        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
-      ]
-    }
-  }
-}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.editorconfig

@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles
 
-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case
 
-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion
 
-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion
 
-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion

.gitattributes

@@ -1,2 +0,0 @@
-*.cmd eol=crlf
-*.sh eol=lf

.gitignore

@@ -8,9 +8,7 @@ cmake-build-debug/
 .idea/
 .dir-locals.el
 *.pyc
-**.vscode/*
-!.vscode/launch.json
-!.vscode/tasks.json
+**.vscode
 .vs
 obj/
 *.csproj.user
@@ -56,6 +54,6 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-.devcontainer/
+**/.devcontainer
 *.Generated.cs
 *.lst

.vscode/launch.json

@@ -1,154 +0,0 @@
-{
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "${workspaceFolder}/build/sparkd-debug-gcc/server/sparkd",
-      "args": ["run"],
-      "stopAtEntry": false,
-      "cwd": "${workspaceFolder}/build/sparkd-debug-gcc/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    
-    {
-      "name": "DataStreamPublisher",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
-      "cwd": "build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DataStreamQueryAnalyzer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-        "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "PubSubServer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
-      "cwd": "build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Parameterized)",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build Spark (Debug)",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "args": [
-          "--sparkd",
-          "${input:sparkConfigServer}"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sparkd",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
-      "program": "",
-      "linux": {
-        "program": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "build/sparkd-debug-clang/server",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    }
-  ],
-  "inputs": [
-    {
-      "id": "dataStreamPublisherConfig",
-      "type": "pickString",
-      "description": "Publisher Configuration",
-      "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
-      ],
-      "default": "MarketDataPublisher",
-    },
-    {
-      "id": "pubSubServerConfig",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
-      ],
-      "default": "PubSubServer",
-    },
-    {
-      "id": "sparkConfigServer",
-      "type": "promptString",
-      "description": "Config Server URL",
-      "default": "http://localhost:8000/config",
-    },
-  ]
-}

.vscode/tasks.json

@@ -1,773 +0,0 @@
-{
-  "version": "2.0.0",
-  "tasks": [
-    {
-      "label": "Build Api Server",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Api Server (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Api Server (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--preset",
-          "${input:sparkdCmakePreset}",
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd -j8",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--preset",
-          "${input:sparkdCmakePreset}",
-          "-j",
-          "8"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "${input:sparkdCompiler}"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug | Gcc)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "gcc"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug | Clang)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "clang"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "${input:sparkdCompiler}"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release | Gcc)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "gcc"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release | Clang)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "clang"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Edit DataStream Config",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "editor",
-        "args": [
-          "${env:HOME}/.local/share/DataStream/conf/spark.conf"
-        ],
-      },
-      "windows": {
-        "command": "vim",
-        "args": [
-          "${env:APPDATA}/bts/DataStream/conf/spark.conf"
-        ],
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "shared",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-    {
-      "label": "Edit Spark Config",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "vim",
-        "args": [
-          "${env:APPDATA}/bts/conf/spark.conf"
-        ],
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "dedicated",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher",
-      "dependsOn": "Build DataStream",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher (Release)",
-      "dependsOn": "Build DataStream (Release)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer",
-      "dependsOn": "Build DataStream",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Release)",
-      "dependsOn": "Build DataStream (Release)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Spark",
-      "dependsOn": "Build Spark",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Release)",
-      "dependsOn": "Build Spark (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark Config Query",
-      "dependsOn": "Build Spark",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "Spark.Console",
-      },
-      "windows": {
-        "command": "Spark.Console.exe",
-      },
-      "args": [
-        "config",
-        "--query",
-        "${input:sparkConfigQuery}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Sparkd Config Query",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "config",
-        "--query",
-        "${input:sparkdConfigQuery}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/logging/show_last_run.cmd",
-        "args": [
-          "DataStream",
-          "DataStreamQueryAnalyzer.exe",
-          "'Starting DataStreamQueryAnalyzer'"
-        ]
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "shared",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-  ],
-  "inputs": [
-    {
-      "id": "buildConfiguration",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "Debug",
-        "Release",
-      ],
-      "default": "Debug",
-    },
-    {
-      "id": "sparkdCmakePreset",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "sparkd-debug-clang",
-        "sparkd-debug-gcc",
-        "sparkd-release-clang",
-        "sparkd-release-gcc",
-      ],
-      "default": "sparkd-debug-clang",
-    },
-    {
-      "id": "sparkdCompiler",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "clang",
-        "gcc",
-      ],
-      "default": "sparkd-debug-clang",
-    },
-    {
-      "id": "dataStreamPublisherConfig",
-      "type": "pickString",
-      "description": "Publisher Configuration",
-      "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
-      ],
-      "default": "MarketDataPublisher",
-    },
-    {
-      "id": "pubSubServerConfig",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
-      ],
-      "default": "PubSubServer",
-    },
-    {
-      "id": "sparkConfigQuery",
-      "type": "promptString",
-      "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkdConfigQuery",
-      "type": "promptString",
-      "description": "Config Type(s)"
-    }
-  ]
-}

CMakeLists.txt

@@ -14,17 +14,12 @@ set(
   CACHE
   STRING
   "External dependencies root folder (should contain folders: lib, bin, include")
-
-# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
-# can run simultaneously without fighting over the same shm files (e.g., sparkd,
-# api server, etc.).
-set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
 set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
 set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
-set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
 if (WIN32)
   set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
   set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -36,13 +31,13 @@ endif()
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})
 
-add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
-if(DEFINED APP_DATA_DIR_OVERRIDE)
-  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
+if(BUILD_API_SERVER)
+  add_compile_definitions(SPARK_ENABLE_API_SERVER)
 endif()
-add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
+
 if(WIN32)
+  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
+  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
   # subdirs
   add_subdirectory(src/lib)
 else()
@@ -52,7 +47,13 @@ else()
   set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
   set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
 
-  if(BUILD_SPARKD)
+  if(BUILD_API_SERVER)
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
+    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
+  elseif(BUILD_SPARKD)
+    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
+
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
     add_compile_definitions(SPARK_ENABLE_CCAPI)

CMakePresets.json

@@ -15,13 +15,11 @@
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build/${presetName}",
+      "binaryDir": "${sourceDir}/build",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -31,13 +29,8 @@
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "spark"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
       }
     },
 
@@ -53,8 +46,6 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
-        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "ON"
@@ -71,8 +62,7 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "sparkd"
+        }
       }
     },
 
@@ -105,72 +95,98 @@
     {
       "name": "sparkd-debug-clang",
       "inherits": "sparkd-debug",
-      "displayName": "Sparkd (Debug | Clang)",
+      "displayName": "Debug (clang)",
       "description": "Debug build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
       }
     },
 
     {
       "name": "sparkd-debug-gcc",
       "inherits": "sparkd-debug",
-      "displayName": "Sparkd (Debug | Gcc)",
+      "displayName": "Debug (gcc)",
       "description": "Debug build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
       }
     },
 
     {
       "name": "sparkd-release-clang",
       "inherits": "sparkd-release",
-      "displayName": "Sparkd (Release | Clang)",
+      "displayName": "Release (clang)",
       "description": "Release build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
       }
     },
 
     {
       "name": "sparkd-release-gcc",
       "inherits": "sparkd-release",
-      "displayName": "Sparkd (Release | Gcc)",
+      "displayName": "Release (gcc)",
       "description": "Release build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
+      }
+    },
+
+    {
+      "name": "spark-api-server-base",
+      "hidden": true,
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
       }
     },
 
     {
       "name": "spark-api-server-windows",
       "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)",
+      "displayName": "Spark Api Server",
       "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
-        "SHARED_MEMORY_PREFIX": "spark_api_server"
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        }
       }
     },
 
     {
       "name": "spark-api-server-linux",
       "inherits": "sparkd",
-      "displayName": "Spark Api Server (Linux)",
+      "displayName": "Spark Api Server",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
         },
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
         "BUILD_SPARKD": {
           "type": "BOOL",
           "value": "OFF"
@@ -181,11 +197,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0"
       }
     },
 
@@ -208,62 +220,13 @@
     },
 
     {
-      "name": "data-stream-suite-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
+      "name": "data-stream-suite",
       "inherits": "spark",
-      "displayName": "Data Stream Suite (Windows)",
+      "displayName": "Data Stream Suite",
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "SHARED_MEMORY_PREFIX": "data_stream"
-      }
-    },
-
-    {
-      "name": "data-stream-suite-linux",
-      "displayName": "Data Stream Suite (Linux)",
-      "description": "Data Stream utils ported over from Edge",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
-      "inherits": "sparkd",
-      "generator": "Ninja Multi-Config",
-      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "BUILD_ADMIN_WEB_APP": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SDK": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SPARKD": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_C_COMPILER": "clang",
-        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
       }
     }
   ],
@@ -281,7 +244,7 @@
 
     {
       "name": "spark-debug",
-      "displayName": "Spark (Debug)",
+      "displayName": "Debug",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Debug"
@@ -289,7 +252,7 @@
 
     {
       "name": "spark-release",
-      "displayName": "Spark (Release)",
+      "displayName": "Release",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Release"
@@ -317,28 +280,28 @@
     {
       "name": "sparkd-debug-clang",
       "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang)",
+      "displayName": "Debug (clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-debug-gcc",
       "configurePreset": "sparkd-debug-gcc",
-      "displayName": "Sparkd (Debug | Gcc)",
+      "displayName": "Debug (gcc)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-clang",
       "configurePreset": "sparkd-release-clang",
-      "displayName": "Sparkd (Release | Clang)",
+      "displayName": "Release (clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-gcc",
       "configurePreset": "sparkd-release-gcc",
-      "displayName": "Sparkd (Release | Gcc)",
+      "displayName": "Release (gcc)",
       "inherits": "sparkd"
     },
 
@@ -355,14 +318,14 @@
 
     {
       "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Windows | Debug)",
+      "displayName": "Spark Api Server (Debug)",
       "inherits": "spark-api-server-windows",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Windows | Release)",
+      "displayName": "Spark Api Server (Debug)",
       "inherits": "spark-api-server-windows",
       "configuration": "Release"
     },
@@ -381,14 +344,14 @@
     {
       "name": "spark-api-server-linux-debug",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Debug)",
+      "displayName": "Spark Api Server (Debug)",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-linux-release",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Release)",
+      "displayName": "Spark Api Server (Release)",
       "configuration": "Release"
     },
 
@@ -408,30 +371,16 @@
     },
 
     {
-      "name": "data-stream-suite-windows-debug",
-      "displayName": "Data Stream Suite (Windows | Debug)",
-      "configurePreset": "data-stream-suite-windows",
+      "name": "data-stream-suite-debug",
+      "displayName": "DataStreamSuite (Debug)",
+      "configurePreset": "data-stream-suite",
       "configuration": "Debug"
     },
 
     {
-      "name": "data-stream-suite-windows-release",
-      "displayName": "Data Stream Suite (Windows | Release)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "data-stream-suite-linux-debug",
-      "displayName": "Data Stream Suite (Linux | Debug)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-linux-release",
-      "displayName": "Data Stream Suite (Linux | Release)",
-      "configurePreset": "data-stream-suite-linux",
+      "name": "data-stream-suite-release",
+      "displayName": "DataStreamSuite (Release)",
+      "configurePreset": "data-stream-suite",
       "configuration": "Release"
     }
   ]

docker/sparkd/dev/Dockerfile

@@ -1,65 +1,74 @@
-FROM ubuntu:latest
+FROM ubuntu:22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
-ENV TZ=America/New_York
 
-# light deps (should be quick to download/install)
-RUN apt-get update && apt-get install -y \
-    ca-certificates \
-    curl \
-    gnupg \
-    liblttng-ust-dev \
-    libpcap0.8-dev \
-    locales \
-    ninja-build \
-    software-properties-common \
-    sqlite3 \
-    tzdata
+# Note installations that take a while (more than 10s)
 
-# clang
-RUN apt-get update && apt-get install -y \
-    clang-13 \
-    clang-format-13 \
-    clangd-13 \
-    clang-15 \
-    clang-format-15 \
-    clangd-15
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
 
-# g++
-RUN apt-get update && apt-get install -y \
-    g++-11 \
-    g++-12
-
-# python
-RUN apt-get update && apt-get install -y \
+# Python: ~30s
+RUN apt-get install -y \
     python3-pip \
     python3-venv \
     && python3 -m pip install --upgrade pip
 
-# dotnet
-RUN apt-get update && apt-get install -y \
-    dotnet7
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-get update \
+    && apt-get install -y g++-$gcc_major
 
-# ssh
-RUN apt-get update && apt-get install -y \
-    libssh2-1-dev \
-    openssh-server
+# Clang
+ARG clang_major=17
+RUN curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+    | tee -a "/etc/apt/sources.list.d/llvm.list" \
+    && apt-get update \
+    && apt-get install -y \
+    clang-$clang_major \
+    clang-format-$clang_major \
+    clangd-$clang_major
 
-# cmake
-RUN \
-    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+# CMake
+RUN curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
     | gpg --dearmor - \
     | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
     && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
     | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
     && apt-get update \
-    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
-    && apt-get install -y kitware-archive-keyring \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
-    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
-# node
+# Dotnet
+RUN curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" \
+    && dpkg -i packages-microsoft-prod.deb \
+    && rm packages-microsoft-prod.deb \
+    && apt-get update \
+    && apt-get install -y dotnet-sdk-8.0
+
+# Node
 ARG node_major=18
 RUN mkdir -p /etc/apt/keyrings \
     && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
@@ -69,60 +78,35 @@ RUN mkdir -p /etc/apt/keyrings \
     && apt-get update \
     && apt-get install -y nodejs
 
-# ccache is done here so that ccache sees clang++ installed, not clang-10
-RUN apt-get update && apt-get install -y \
-    ccache 
-
-# emumba (light deps)
+# Emumba deps
 RUN apt-get update && apt-get install -y \
     build-essential \
     doxygen \
-    python3-sphinx \
     graphviz \
     libgtest-dev \
     libssl-dev \
+    pkg-config \
+    python3-sphinx \
     rapidjson-dev \
-    zlib1g-dev \
-    pkg-config
+    zlib1g-dev
 
-# emumba (boost)
-RUN apt-get update && apt-get install -y \
-    libboost-filesystem-dev \
-    libboost-program-options-dev \
-    libboost-test-dev \
-    libboost-date-time-dev
-
-# dev utils
-RUN apt-get update && apt-get install -y \ 
-    fish \
-    git-all \
-    sudo \
-    vim \
-    wget
-
-# set up default executables (update-alternatives)
-RUN \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
-    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+# Set up default executables (update-alternatives)
+RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
 
 # Add compiler symlinks for ccache 
-RUN \
-    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang; \
     ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc; \
     ln -s /usr/bin/ccache /usr/local/bin/g++; \
-    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
-    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
-    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
-    ln -s /usr/bin/ccache /usr/local/bin/g++-12
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-$gcc_major; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-$gcc_major
 
 RUN locale-gen en_US.UTF-8
 
@@ -130,8 +114,7 @@ ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
 
-RUN \
-    groupadd -g "$docker_uid" "$docker_user" \
+RUN groupadd -g "$docker_uid" "$docker_user" \
     && useradd \
     -m \
     -u "$docker_uid" \

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

playground-1.mongodb.js

@@ -0,0 +1,45 @@
+/* global use, db */
+// MongoDB Playground
+// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
+// Make sure you are connected to enable completions and to be able to run a playground.
+// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
+// The result of the last command run in a playground is shown on the results panel.
+// By default the first 20 documents will be returned with a cursor.
+// Use 'console.log()' to print to the debug output.
+// For more documentation on playgrounds please refer to
+// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
+
+// Select the database to use.
+use('mongodbVSCodePlaygroundDB');
+
+// Insert a few documents into the sales collection.
+db.getCollection('sales').insertMany([
+    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
+    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
+    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
+    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
+    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
+    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
+    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
+    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
+]);
+
+// Run a find command to view items sold on April 4th, 2014.
+const salesOnApril4th = db.getCollection('sales').find({
+    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
+}).count();
+
+// Print a message to the output window.
+console.log(`${salesOnApril4th} sales occurred in 2014.`);
+
+// Here we run an aggregation and open a cursor to the results.
+// Use '.toArray()' to exhaust the cursor to return the whole result set.
+// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
+db.getCollection('sales').aggregate([
+    // Find all of the sales that occurred in 2014.
+    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
+    // Group the total sales for each product.
+    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
+]);
+
+

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/Crono.cs

@@ -1,22 +1,41 @@
-using System.Diagnostics;
+using mbts.spark;
 
-namespace BCG.Utilities.General;
+using System;
+using System.Runtime;
 
-/// <summary>
-/// Summary description for Crono.
-/// </summary>
-public class Crono {
-   readonly Stopwatch myTimer = new();
+namespace BCG.Utilities.General
+{
+	/// <summary>
+	/// Summary description for Crono.
+	/// </summary>
+	public class Crono
+	{
+      public Crono()
+      {
+         myTimer = new HighPerformanceTimer();
+      }
 
-   public void start() =>
-      myTimer.Start();
+      public void start() 
+      {myTimer.start();}
 
-   public void stop() =>
-      myTimer.Stop();
+      public void stop()
+      {myTimer.stop();}
 
-   public TimeSpan time =>
-      myTimer.Elapsed;
+      public TimeSpan time
+      {
+         get 
+         {
+            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
+         }
+      }
+
+      public double miliseconds
+      {
+         get
+         {return myTimer.totalMilliseconds;}
+      }
+
+      private HighPerformanceTimer myTimer;
+	}
 
-   public double miliseconds =>
-      myTimer.Elapsed.TotalMilliseconds;
 }

src/DataStreamSuite/BlueRegistry/BlueRegistry.cs

@@ -1,192 +1,192 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Reflection;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public enum RegistryRoot { HKCU, HKLM }
-
-   [Serializable]
-   public class BlueRegistry
-   {
-      public const string BlueSubKey = @"Software\Blue Capital Group\";
-      public static readonly string ApplicationPrefix;
-      protected string myFullPath;
-
-      public BlueRegistry(string subPath)
-      {
-         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
-      }
-
-      public BlueRegistry(BlueRegistry parent, string subPath)
-      {
-         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
-      }
-
-      static BlueRegistry()
-      {
-         string app = "Applications";
-
-         Assembly ass = Assembly.GetEntryAssembly();
-         if (ass != null)
-         {
-            app = ass.GetName().Name;
-
-            object[] attrs = ass.GetCustomAttributes(
-               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
-
-            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
-            {
-               app = ((AssemblyTitleAttribute)attrs[0]).Title;
-            }
-         }
-
-         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
-      }
-
-      public object getValue(string name)
-      {
-         return getValue(RegistryRoot.HKCU, name);
-      }
-
-      public object getValue(string name, object defaultValue)
-      {
-         return getValue(RegistryRoot.HKCU, name, defaultValue);
-      }
-
-      public void setValue(string name, object value)
-      {
-         setValue(RegistryRoot.HKCU, name, value);
-      }
-
-      public void deleteValue(string name)
-      {
-         deleteValue(RegistryRoot.HKCU, name);
-      }
-
-      private RegistryKey myGetRegistryKey(RegistryRoot root)
-      {
-         if (root == RegistryRoot.HKCU)
-            return Registry.CurrentUser.CreateSubKey(myFullPath);
-         else
-            return Registry.LocalMachine.CreateSubKey(myFullPath);
-      }
-
-      #region Generic Methods
-      public virtual void setValue(RegistryRoot root, string name, object value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.SetValue(name, value);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name);
-         }
-      }
-
-      internal virtual void deleteValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.DeleteValue(name);
-         }
-      }
-
-      public virtual object getValue(RegistryRoot root, string name, object defaultValue)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name, defaultValue);
-         }
-      }
-      #endregion
-
-      #region Double
-      internal virtual void setDouble(RegistryRoot root, string name, double value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            DoubleRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual double getDouble(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return DoubleRegistryHandler.getValue(rk, name);
-         }
-      }
-      #endregion
-
-      #region Enum
-      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            EnumRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return EnumRegistryHandler.getValue(rk, name, enumType);
-         }
-      }
-      #endregion
-
-      #region Array
-      internal virtual void setArray(RegistryRoot root, string name, Array array)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            ArrayRegistryHandler.setValue(rk, name, array);
-         }
-      }
-
-      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return ArrayRegistryHandler.getValue(rk, name, elementType);
-         }
-      }
-      #endregion
-
-      #region SortedList
-      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.setValue(rk, name, slist);
-         }
-      }
-
-      internal virtual SortedList getSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return SortedListRegistryHandler.getValue(rk, name);
-         }
-      }
-
-      internal virtual void deleteSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.deleteValue(rk, name);
-         }
-      }
-      #endregion
-   }
-}
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      internal virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,9 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>
 
 </Project>

src/DataStreamSuite/BlueRegistry/PersistWindowState.cs

@@ -1,166 +1,166 @@
-using System;
-using System.Reflection;
-using System.Windows.Forms;
-using System.Drawing;
-using System.Diagnostics;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public class PersistWindowState
-   {
-      // event info that allows form to persist extra window state data
-      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
-      public event WindowStateHandler LoadState;
-      public event WindowStateHandler SaveState;
-
-      private Form myParent;
-      private BlueRegistry myRegistry;
-
-      private int myNormalLeft;
-      private int myNormalTop;
-      private int myNormalWidth;
-      private int myNormalHeight;
-      private bool myResizable = true;
-      private FormWindowState myWindowState;
-      private bool myAllowSaveMinimized = false;
-      private bool mySavePosition = true;
-      private bool mySaveState = true;
-
-      public PersistWindowState (bool savePosition)
-      {
-         mySavePosition = savePosition;
-      }
-
-      public PersistWindowState ()
-      {
-      }
-
-      public bool saveState
-      {
-         [DebuggerStepThrough]
-         set { mySaveState = value; }
-      }
-
-      public Form parent
-      {
-         [DebuggerStepThrough]
-         get { return myParent; }
-         set
-         {
-            myParent = value;
-            myRegistry = new BlueRegistry (myParent.Text);
-
-            // subscribe to parent form's events
-            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
-            myParent.Resize += new System.EventHandler(myOnResize);
-            myParent.Move += new System.EventHandler(myOnMove);
-            myParent.Load += new System.EventHandler(myOnLoad);
-
-            // get initial width and height in case form is never resized
-            myNormalWidth = myParent.Width;
-            myNormalHeight = myParent.Height;
-         }
-      }
-
-      public BlueRegistry blueRegistry
-      {
-         [DebuggerStepThrough]
-         get { return myRegistry; }
-         [DebuggerStepThrough]
-         set { myRegistry = value; }
-      }
-
-      public bool resizable
-      {
-         [DebuggerStepThrough]
-         set { myResizable = value; }
-      }
-
-      public bool allowSaveMinimized
-      {
-         [DebuggerStepThrough]
-         set { myAllowSaveMinimized = value; }
-      }
-
-      private void myOnResize(object sender, System.EventArgs e)
-      {
-         // save width and height
-         if (myParent.WindowState == FormWindowState.Normal && myResizable)
-         {
-            myNormalWidth = myParent.Width;
-            myNormalHeight = myParent.Height;
-         }
-      }
-
-      private void myOnMove(object sender, System.EventArgs e)
-      {
-         // save position
-         if(myParent.WindowState == FormWindowState.Normal)
-         {
-            myNormalLeft = myParent.Left;
-            myNormalTop = myParent.Top;
-         }
-         // save state
-         myWindowState = myParent.WindowState;
-      }
-
-      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
-      {
-         // save position, size and state
-         if (mySavePosition)
-         {
-            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
-            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
-         }
-
-         if (myResizable)
-         {
-            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
-            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
-         }
-
-         // check if we are allowed to save the state as minimized (not normally)
-         if (!myAllowSaveMinimized)
-         {
-            if(myWindowState == FormWindowState.Minimized)
-               myWindowState = FormWindowState.Normal;
-         }
-
-         if (mySaveState)
-            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
-        
-         // fire SaveState event
-         if (SaveState != null)
-            SaveState (this, myRegistry);
-      }
-
-      private void myOnLoad (object sender, System.EventArgs e)
-      {
-         if (mySavePosition)
-         {
-            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
-            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
-            myParent.Location = new Point (left, top);
-         }
-
-         if (myResizable)
-         {
-            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
-            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
-            myParent.Size = new Size (width, height);
-         }
-            
-         if (mySaveState)
-         {
-            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
-               (int) myParent.WindowState);
-            myParent.WindowState = windowState;
-         }
-
-         // fire LoadState event
-         if(LoadState != null)
-            LoadState (this, myRegistry);
-      }
-   }
-}
+using System;
+using System.Reflection;
+using System.Windows.Forms;
+using System.Drawing;
+using System.Diagnostics;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public class PersistWindowState
+   {
+      // event info that allows form to persist extra window state data
+      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
+      public event WindowStateHandler LoadState;
+      public event WindowStateHandler SaveState;
+
+      private Form myParent;
+      private BlueRegistry myRegistry;
+
+      private int myNormalLeft;
+      private int myNormalTop;
+      private int myNormalWidth;
+      private int myNormalHeight;
+      private bool myResizable = true;
+      private FormWindowState myWindowState;
+      private bool myAllowSaveMinimized = false;
+      private bool mySavePosition = true;
+      private bool mySaveState = true;
+
+      public PersistWindowState (bool savePosition)
+      {
+         mySavePosition = savePosition;
+      }
+
+      public PersistWindowState ()
+      {
+      }
+
+      public bool saveState
+      {
+         [DebuggerStepThrough]
+         set { mySaveState = value; }
+      }
+
+      public Form parent
+      {
+         [DebuggerStepThrough]
+         get { return myParent; }
+         set
+         {
+            myParent = value;
+            myRegistry = new BlueRegistry (myParent.Text);
+
+            // subscribe to parent form's events
+            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
+            myParent.Resize += new System.EventHandler(myOnResize);
+            myParent.Move += new System.EventHandler(myOnMove);
+            myParent.Load += new System.EventHandler(myOnLoad);
+
+            // get initial width and height in case form is never resized
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      public BlueRegistry blueRegistry
+      {
+         [DebuggerStepThrough]
+         get { return myRegistry; }
+         [DebuggerStepThrough]
+         set { myRegistry = value; }
+      }
+
+      public bool resizable
+      {
+         [DebuggerStepThrough]
+         set { myResizable = value; }
+      }
+
+      public bool allowSaveMinimized
+      {
+         [DebuggerStepThrough]
+         set { myAllowSaveMinimized = value; }
+      }
+
+      private void myOnResize(object sender, System.EventArgs e)
+      {
+         // save width and height
+         if (myParent.WindowState == FormWindowState.Normal && myResizable)
+         {
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      private void myOnMove(object sender, System.EventArgs e)
+      {
+         // save position
+         if(myParent.WindowState == FormWindowState.Normal)
+         {
+            myNormalLeft = myParent.Left;
+            myNormalTop = myParent.Top;
+         }
+         // save state
+         myWindowState = myParent.WindowState;
+      }
+
+      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
+      {
+         // save position, size and state
+         if (mySavePosition)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
+            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
+         }
+
+         if (myResizable)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
+            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
+         }
+
+         // check if we are allowed to save the state as minimized (not normally)
+         if (!myAllowSaveMinimized)
+         {
+            if(myWindowState == FormWindowState.Minimized)
+               myWindowState = FormWindowState.Normal;
+         }
+
+         if (mySaveState)
+            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
+        
+         // fire SaveState event
+         if (SaveState != null)
+            SaveState (this, myRegistry);
+      }
+
+      private void myOnLoad (object sender, System.EventArgs e)
+      {
+         if (mySavePosition)
+         {
+            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
+            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
+            myParent.Location = new Point (left, top);
+         }
+
+         if (myResizable)
+         {
+            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
+            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
+            myParent.Size = new Size (width, height);
+         }
+            
+         if (mySaveState)
+         {
+            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
+               (int) myParent.WindowState);
+            myParent.WindowState = windowState;
+         }
+
+         // fire LoadState event
+         if(LoadState != null)
+            LoadState (this, myRegistry);
+      }
+   }
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -42,7 +42,6 @@ OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
 Negate                   '!'
-MapsTo                   ':'
 IsOfType                 '/'
 
 %%
@@ -98,10 +97,6 @@ IsOfType                 '/'
    return (int)Token.NEGATE;
 }
 
-{MapsTo} {
-   return (int)Token.MAPS_TO;
-}
-
 {Equal} {
    return (int)Token.EQUAL;
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.DataStream.Websocket/ClientSubscription.cs

@@ -0,0 +1,9 @@
+namespace Bts.DataStream.Websocket;
+
+class ClientSubscription(WebsocketClientID id, SubscriptionParams @params, IDisposable dispose) : IDisposable {
+   public SubscriptionParams Params => @params;
+   public WebsocketClientID Id => id;
+
+   public void Dispose() =>
+      dispose.Dispose();
+}

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -0,0 +1,301 @@
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.collections;
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+using Spark.Core.Json;
+
+using System.Buffers;
+using System.Reactive.Subjects;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+/// <summary>
+/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
+/// requests (from clients) as input, and emits data/notifications (to clients) as output.
+/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
+/// </summary>
+public class DataSync(JsonSerializerOptions? serializerOpts = null) :
+   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
+   protected readonly object lockObj = new();
+   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+
+   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
+
+   public void OnCompleted() =>
+      Log.Api.info("DataSync incoming request stream has completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"DataSync error: {error}");
+
+   public void OnNext(WebsocketConnectionUpdate value) {
+      switch (value.Type) {
+         case WebsocketConnectionUpdateType.IncomingMessage:
+            onIncomingMessage(value.IncomingMessage);
+            break;
+         case WebsocketConnectionUpdateType.Connect:
+            onConnect(value.Connect);
+            break;
+         case WebsocketConnectionUpdateType.Disconnect:
+            onDisconnect(value.Disconnect);
+            break;
+         default:
+            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
+            break;
+      }
+   }
+
+   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
+      lock (lockObj) {
+         requestSend += observer.OnNext;
+         return Disposable.Create(Dispose);
+      }
+      void Dispose() {
+         lock (lockObj) {
+            requestSend -= observer.OnNext;
+         }
+      }
+   }
+
+   static JsonRpcId? extractRpcId(string messageBody) {
+      try {
+         return JsonNode.Parse(messageBody)
+            ?.AsObject()
+            ?.GetValueOrDefault("id")
+            ?.Deserialize<JsonRpcId>();
+      } catch {
+         return null;
+      }
+   }
+
+   static string getRpcNotification(DataPacket dp) {
+      using (var buffer = new MemoryStream())
+      using (var writer = new Utf8JsonWriter(buffer))
+      using (writer.WriteObject()) {
+         writer.WriteString("method", "subscription");
+         using (writer.WriteObject("params")) {
+            writer.Write("tableName", dp.schema.tableName);
+            writer.Write("columns", dp.schema.columnInfo);
+            var cellJsonFactories = dp.schema.columnInfo.Select(getCellJsonFactory).ToList();
+            using (writer.WriteArray("added")) {
+               foreach (IBlueRow row in (dp.addedRows?.Cast<IBlueRow>()).empty_if_null()) {
+                  using (writer.WriteObject()) {
+                     for (int i = 0; i < cellJsonFactories.Count; i++) {
+                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
+                     }
+                  }
+               }
+            }
+            using (writer.WriteArray("deleted")) {
+               foreach (IBlueRow row in (dp.deletedRows?.Cast<IBlueRow>()).empty_if_null()) {
+                  using (writer.WriteObject()) {
+                     for (int i = 0; i < cellJsonFactories.Count; i++) {
+                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
+                     }
+                  }
+               }
+            }
+         }
+         writer.WriteString("jsonrpc", "2.0");
+         writer.Flush();
+         return Encoding.UTF8.GetString(buffer.ToArray());
+      }
+   }
+
+   static Func<IBlueRow, JsonNode?> getCellJsonFactory(ColumnInfo colInfo) {
+      var t = colInfo.colType;
+      if (t == typeof(int)) {
+         return NullCheckWrapper(v => (int)v);
+      }
+      if (t == typeof(uint)) {
+         return NullCheckWrapper(v => (uint)v);
+      }
+      if (t == typeof(long)) {
+         return NullCheckWrapper(v => (long)v);
+      }
+      if (t == typeof(ulong)) {
+         return NullCheckWrapper(v => (ulong)v);
+      }
+      if (t == typeof(short)) {
+         return NullCheckWrapper(v => (short)v);
+      }
+      if (t == typeof(ushort)) {
+         return NullCheckWrapper(v => (ushort)v);
+      }
+      if (t == typeof(double)) {
+         return NullCheckWrapper(v => (double)v);
+      }
+      if (t == typeof(float)) {
+         return NullCheckWrapper(v => (float)v);
+      }
+      if (t == typeof(bool)) {
+         return NullCheckWrapper(v => (bool)v);
+      }
+      return NullCheckWrapper(v => (string)v);
+
+      Func<IBlueRow, JsonNode?> NullCheckWrapper(Func<object, JsonNode> toJson) =>
+         r => {
+            var obj = r[colInfo.colNo];
+            if (obj is null) {
+               return null;
+            }
+            return toJson(obj);
+         };
+   }
+
+   void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, []);
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   void onIncomingMessage(WebsocketIncomingMessage message) {
+      var id = message.sender_client_id;
+      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
+         onBadRequest(message, err);
+         return;
+      }
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "publish":
+            Log.Api.error("DataStreamWebsocket api 'publish' not implemented.");
+            // TODO
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
+      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
+      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
+      lock (lockObj) {
+         if (requestSend is null) {
+            Log.Api.warn(
+               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
+            return false;
+         }
+         requestSend(messages);
+         return true;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += OnChange;
+      }
+
+      void OnChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var notif = getRpcNotification(dp);
+         send([new(id, notif)]);
+      }
+
+      void Dispose() =>
+         stream.onChange -= OnChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
+      var rpcId = extractRpcId(req.message_body);
+      var logMsg = $"Recieved invalid request:\n{req.message_body}";
+      if (errorMsg is not null) {
+         logMsg += $"\n\n**Exception**\n{errorMsg}";
+      }
+      Log.Api.error(logMsg);
+      if (rpcId is null) {
+         return;
+      }
+      // Send error back to client
+      var rpcErr = JsonRpcError.Create(
+         (long)JsonRpcErrorCode.InvalidRequest,
+         $"Received invalid request: {errorMsg ?? "unknown error"}",
+         req.message_body);
+      sendError(req.sender_client_id, rpcId.Value, rpcErr);
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/JObjectExtensions.cs

@@ -0,0 +1,34 @@
+namespace Bts.DataStream.Websocket;
+
+using Newtonsoft.Json.Linq;
+
+using System.Diagnostics.CodeAnalysis;
+
+public static class JObjectExtensions {
+   public static bool TryParse<T>(
+      this JObject filter, string key, [MaybeNull] out T obj, [NotNullWhen(false)] out string? error) {
+      obj = default;
+      var value = filter.GetValueOrDefault(key)?.ToString();
+      if (value == null) {
+         error = $"{key} not present";
+         return true;
+      }
+      return WebsocketJsonParsers.TryParse(value, key, out obj, out error);
+   }
+
+   public static bool TryParse<T>(
+     this JObject filter, string key, out T result, [NotNullWhen(false)] out string? error, T defaultValue) {
+      var value = filter.GetValueOrDefault(key)?.ToString();
+      if (value == null) {
+         result = defaultValue;
+         error = $"{key} not present";
+         return true;
+      }
+      if (WebsocketJsonParsers.TryParse<T>(value, key, out var maybeRes, out error)) {
+         result = maybeRes ?? defaultValue;
+         return true;
+      }
+      result = defaultValue;
+      return false;
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs

@@ -0,0 +1,48 @@
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class MultilineStringJsonConverter : JsonConverter<string> {
+   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (typeToConvert != typeof(string)) {
+         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+      }
+      return reader.TokenType switch {
+         JsonTokenType.String => StringImpl(ref reader),
+         JsonTokenType.StartArray => ArrayImpl(ref reader),
+         _ => throw new ArgumentException(
+            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+      };
+
+      static string? StringImpl(ref Utf8JsonReader reader) =>
+         new(reader.GetString()!);
+
+      static string? ArrayImpl(ref Utf8JsonReader reader) {
+         var lines = new List<string>(2);
+         while (reader.Read()) {
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  lines.Add(reader.GetString()!);
+                  break;
+               case JsonTokenType.EndArray:
+                  return string.Join("\n", lines);
+            }
+         }
+         throw new JsonException(
+            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
+      var lines = value.Split("\n");
+      if (lines.Length < 2) {
+         writer.WriteRawValue(value);
+         return;
+      }
+      writer.WriteRawValue($"[{", ".join(lines)}]");
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/RequestType.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStream.Websocket;
+
+public enum RequestType {
+   Unspecified,
+   Subscribe,
+   Unsubscribe,
+   UnsubscribeAll,
+}

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -0,0 +1,25 @@
+namespace Bts.DataStream.Websocket;
+
+using bts;
+
+using Google.Protobuf;
+
+using System.Text.Json;
+
+public static class Session {
+   static int sessionIdCounter;
+
+   public static async Task Start(
+      WebsocketServerConfig config, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      var sessionId = Interlocked.Increment(ref sessionIdCounter);
+      var gateway = new WebsocketClient(config);
+      var sync = new DataSync(opts);
+      using (gateway.Subscribe(sync))
+      using (sync.Subscribe(gateway)) {
+         Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
+         Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
+         await gateway.ConnectAsync();
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -0,0 +1,112 @@
+namespace Bts.DataStream.Websocket;
+
+using bts.utils;
+
+using Spark.Core;
+using Spark.Core.Json;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionParams> {
+   static class PropNames {
+      public const string QUERY = "query";
+   }
+
+   // Serializer supports plain string, or an array of lines for convenience
+   [MaybeNull]
+   [JsonPropertyName(PropNames.QUERY)]
+   [JsonConverter(typeof(MultilineStringJsonConverter))]
+   public string? Query { get; set; } = query;
+
+   public bool Equals(SubscriptionParams? other) =>
+      other is not null &&
+      // Not the most elegant equals impl, but it'll work
+      Query.safe_equals(other.Query, (a, b) => a?.ToString() == b?.ToString());
+
+   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
+      public override bool CanConvert(Type typeToConvert) =>
+         typeToConvert == typeof(SubscriptionParams);
+
+      public override SubscriptionParams Read(
+         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+         if (typeToConvert != typeof(SubscriptionParams)) {
+            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+         }
+         return read(ref reader);
+      }
+
+      static SubscriptionParams read(ref Utf8JsonReader reader) {
+         return reader.TokenType switch {
+            JsonTokenType.String => StringImpl(ref reader),
+            JsonTokenType.StartArray => ArrayImpl(ref reader),
+            JsonTokenType.StartObject => ObjectImpl(ref reader),
+            _ => throw new ArgumentException(
+               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+         };
+
+         static SubscriptionParams StringImpl(ref Utf8JsonReader reader) =>
+            new(reader.GetString()!);
+
+         static SubscriptionParams ArrayImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+
+         static SubscriptionParams ObjectImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.PropertyName:
+                     var propName = reader.GetString()!;
+                     if (propName == PropNames.QUERY) {
+                        switch (reader.TokenType) {
+                           case JsonTokenType.StartArray:
+                              return ArrayImpl(ref reader);
+                           case JsonTokenType.String:
+                              return StringImpl(ref reader);
+                        }
+                     }
+                     break;
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+      }
+
+      public override void Write(
+         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
+         var q = value.Query;
+         var lines = q?.Split("\n");
+         if (lines is null || lines.Length < 2) {
+            writer.WriteString(PropNames.QUERY, value.Query);
+            return;
+         }
+         writer.WriteArray(PropNames.QUERY, lines);
+      }
+   }
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as SubscriptionParams);
+
+   public override int GetHashCode() =>
+      Query?.Length ?? typeof(SubscriptionParams).GetHashCode();
+}

src/DataStreamSuite/Bts.DataStream.Websocket/Usings.cs

@@ -0,0 +1,5 @@
+global using bts.websocket_core;
+
+global using System;
+
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs

@@ -0,0 +1,41 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class ViewStreamUpdate {
+   [JsonPropertyName("tableName")]
+   public string? TableName { get; set; }
+
+   [JsonPropertyName("columns")]
+   public List<ColumnInfo>? Columns { get; set; }
+
+   [JsonPropertyName("addedRows")]
+   public List<Dictionary<string, object?>>? AddedRows { get; set; }
+
+   [JsonPropertyName("deletedRows")]
+   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
+
+   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
+      var schema = new BlueTableSchema([.. Columns!], TableName);
+      var addedRowsTable = new BlueTable(schema);
+      var deletedRowsTable = new BlueTable(schema);
+      if (applyChanges) {
+         state.deleteRows(deletedRowsTable);
+         state.addRows(addedRowsTable);
+      }
+      return new(state, addedRowsTable, deletedRowsTable);
+   }
+
+   public static string ToJson() {
+      return "";
+   }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStream.Websocket\Bts.DataStream.Websocket.csproj" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServer.conf

@@ -0,0 +1,7 @@
+{
+  "host": "localhost",
+  "port": 8080,
+  "scheme": "ws",
+  "default_end_point": "/",
+  "keep_alive_interval_secs": 1
+}

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs

@@ -0,0 +1,76 @@
+using bts.api.core;
+
+using Bts.DataStream.Websocket;
+
+using Google.Protobuf;
+
+using System.CommandLine.Parsing;
+using System.Text.Json;
+
+static WebsocketServerConfig readConfigFile(string path) {
+   using var fs = File.OpenRead(path);
+   using var sr = new StreamReader(fs);
+   return JsonParser.Default.Parse<WebsocketServerConfig>(sr);
+}
+
+static WebsocketServerConfig? readConfigArg(ArgumentResult result) {
+   var path = result.Tokens.Select(t => t.Value).FirstOrDefault() ?? string.Empty;
+   return readConfigFile(path);
+}
+
+static WebsocketServerConfig getDefaultConfig() {
+   const string defaultCfgPath = "DataStreamWebsocketServer.conf";
+   if (File.Exists(defaultCfgPath)) {
+      return readConfigFile(defaultCfgPath);
+   }
+   return new() {
+      Scheme = "http",
+      Host = "localhost",
+      Port = 8080,
+      DefaultEndPoint = "/",
+      KeepAliveIntervalSecs = 1
+   };
+}
+   
+static async Task run(
+   WebsocketServerConfig? wssConfig, string user, string password, string? sparkdConfigSource) {
+   var opts = new JsonSerializerOptions(JsonSerializerDefaults.Web);
+   wssConfig ??= getDefaultConfig();
+   Console.WriteLine("Starting DataStreamWebsocketServer...");
+   var serviceCtx = ServiceContext.create(sparkdConfigSource);
+   await serviceCtx.login(user, password);
+   await Session.Start(wssConfig, opts);
+   // if (sparkdConfigSource is not null && sparkdConfigSource.Length > 9999) {
+   // } else {
+   //    await ChatGptSession.Start();
+   // }
+}
+
+var userOpt = new Option<string>("--user", () => "bts");
+var passOpt = new Option<string>("--password", () => string.Empty);
+var sparkdConfigSourceOpt = new Option<string>(
+   "--sparkd", description: "The sparkd config server url");
+// var bling = new ParseArgument<WebsocketServerConfig?>(parseWsConfig);
+
+// WebsocketServerConfig? parseWsConfig(ArgumentResult result) {
+//    Token t;
+//    TokenType tt;
+//    Symbol s;
+//    throw new NotImplementedException();
+// }
+
+var configOpt = new Option<WebsocketServerConfig?>(
+   "--config", readConfigArg, description: "WebsocketServerConfig file path");
+configOpt.SetDefaultValue(new());
+var rootCommand = new RootCommand("Starts a DataStream Websocket Server");
+rootCommand.AddOption(configOpt);
+rootCommand.AddOption(userOpt);
+rootCommand.AddOption(passOpt);
+rootCommand.SetHandler(run, configOpt, userOpt, passOpt, sparkdConfigSourceOpt);
+
+try {
+   return await rootCommand.InvokeAsync(args);
+} catch (Exception ex) {
+   Console.WriteLine($"Error: {ex}");
+}
+return 1;

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs

@@ -0,0 +1,6 @@
+global using bts.websocket_core;
+global using bts.websocket_core.json_rpc;
+
+global using System.CommandLine;
+
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.2' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>false</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\dist</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-use-websocket": "^4.5.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.2.43",
+    "@types/react-dom": "^18.2.17",
+    "@vitejs/plugin-react": "^4.2.1",
+    "eslint": "^8.55.0",
+    "eslint-plugin-react": "^7.33.2",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.5",
+    "vite": "^5.0.8"
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -0,0 +1,44 @@
+#root {
+    max-width: 1280px;
+    margin: 0 auto;
+    padding: 2rem;
+    text-align: center;
+    --cell-background-color: #293136;
+    --cell-color: #cbe3ef;
+}
+
+tr:nth-child(even) {
+    background: #F2F2F2;
+}
+
+tr:nth-child(odd) {
+    background: #FFF;
+}
+
+th, td {
+    padding-left: 1rem;
+    padding-right: 1rem;
+    background-color: var(--cell-background-color);
+    color: var(--cell-color);
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -0,0 +1,88 @@
+import { Component } from "react";
+import "./App.css";
+
+export class Home extends Component {
+  static displayName = Home.name;
+  static sparkdHost = "localhost";
+
+  render() {
+    let socket = null;
+    const wsUrl = `ws://localhost:5268/ws`;
+
+    function connectToSparkd() {
+      if (socket == null) {
+        socket = new WebSocket(wsUrl);
+      }
+      if (socket && socket.readyState === WebSocket.OPEN) {
+        return;
+      }
+
+      const output = document.getElementById("query-output");
+      socket.onopen = (event) => {
+        console.log(`Connected to ${wsUrl}`);
+      };
+
+      socket.onmessage = (event) => {
+        console.log(`Received message: ${event.data}`);
+        output.innerHTML = event.data;
+      };
+
+      socket.onerror = (error) => {
+        console.log(`Error on websocket ${wsUrl}: ${error}`);
+      };
+
+      socket.onclose = (event) => {
+        if (event.wasClean) {
+          const msg = `Connection to ${wsUrl} closed cleanly, code=${event.code}, reason=${event.reason}`;
+          console.log(msg);
+        } else {
+          const msg = `Connection to ${wsUrl} abruptly closed`;
+          console.log(msg);
+        }
+      };
+    }
+
+    function executeQuery() {
+      if (socket == null || !socket) {
+        connectToSparkd();
+      }
+      if (socket == null || !socket) {
+        console.log("Failed to connect to websocket server");
+        return;
+      }
+      if (socket.readyState === WebSocket.OPEN) {
+        const queryInput = document.getElementById("query-input");
+        const rpcReq = {
+          jsonrpc: "2.0",
+          method: "subscribe",
+          params: {
+            query: queryInput.value,
+          },
+          id: 1,
+        };
+        socket.send(JSON.stringify(rpcReq));
+      }
+    }
+
+    return (
+      <div>
+        <h1>Data Stream Query Analyzer</h1>
+        <p>Run queries against the BTS Cloud</p>
+        <div>
+          <textarea id="query-input" placeholder="Enter Query"></textarea>
+          <button id="query-button" onClick={executeQuery}>
+            Execute
+          </button>
+        </div>
+        <table id="query-output"></table>
+        <p id="footer">
+          The <code>ClientApp</code> subdirectory is a standard React
+          application based on the <code>create-react-app</code> template. If
+          you open a command prompt in that directory, you can run{" "}
+          <code>npm</code> commands such as <code>npm test</code> or{" "}
+          <code>npm install</code>.
+        </p>
+      </div>
+    );
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -0,0 +1,68 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -0,0 +1,10 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import {Home} from './App.jsx'
+import './index.css'
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <Home />
+  </React.StrictMode>,
+)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -0,0 +1,60 @@
+import { fileURLToPath, URL } from 'node:url';
+
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+
+const baseFolder =
+    process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+        ? `${process.env.APPDATA}/ASP.NET/https`
+        : `${process.env.HOME}/.aspnet/https`;
+
+const certificateArg = process.argv.map(arg => arg.match(/--name=(?<value>.+)/i)).filter(Boolean)[0];
+const certificateName = certificateArg ? certificateArg.groups.value : "bts.datastreamwebapp.client";
+
+if (!certificateName) {
+    console.error('Invalid certificate name. Run this script in the context of an npm/yarn script or pass --name=<<app>> explicitly.')
+    process.exit(-1);
+}
+
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+    if (0 !== child_process.spawnSync('dotnet', [
+        'dev-certs',
+        'https',
+        '--export-path',
+        certFilePath,
+        '--format',
+        'Pem',
+        '--no-password',
+    ], { stdio: 'inherit', }).status) {
+        throw new Error("Could not create certificate.");
+    }
+}
+
+// https://vitejs.dev/config/
+export default defineConfig({
+    plugins: [plugin()],
+    resolve: {
+        alias: {
+            '@': fileURLToPath(new URL('./src', import.meta.url))
+        }
+    },
+    server: {
+        proxy: {
+            '^/weatherforecast': {
+                target: 'https://localhost:7128/',
+                secure: false
+            }
+        },
+        port: 5173,
+        https: {
+            key: fs.readFileSync(keyFilePath),
+            cert: fs.readFileSync(certFilePath),
+        }
+    }
+})

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy">
+      <Version>8.*-*</Version>
+    </PackageReference>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.http

@@ -0,0 +1,6 @@
+@Bts.DataStreamWebApp.Server_HostAddress = http://localhost:5268
+
+GET {{Bts.DataStreamWebApp.Server_HostAddress}}/weatherforecast/
+Accept: application/json
+
+###

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -0,0 +1,107 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.api.core;
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using Microsoft.AspNetCore.Mvc;
+
+using Spark.Core.Disposables;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
+   readonly ILogger<DataStreamSubscriptionController> logger = logger;
+   IDisposable? streamSubscription;
+
+   [Route("/ws")]
+   public async Task Get() {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      Console.WriteLine("Establishing websocket connection...");
+      using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+      Console.WriteLine($"Websocket connection established.");
+      await runWebSocketServer(null, "bts", string.Empty, webSocket);
+   }
+
+   public static async Task Echo(WebSocket webSocket) {
+      var buffer = new byte[1024 * 4];
+      var receiveResult = await webSocket.ReceiveAsync(
+          new ArraySegment<byte>(buffer), CancellationToken.None);
+
+      while (!receiveResult.CloseStatus.HasValue) {
+         await webSocket.SendAsync(
+             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+
+         receiveResult = await webSocket.ReceiveAsync(
+             new ArraySegment<byte>(buffer), CancellationToken.None);
+      }
+
+      await webSocket.CloseAsync(
+          receiveResult.CloseStatus.Value,
+          receiveResult.CloseStatusDescription,
+          CancellationToken.None);
+   }
+
+   async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
+      var context = ServiceContext.create(sparkd);
+      await context.login(user, password);
+      var buffer = new byte[1024 * 4];
+      var receiveResult = await webSocket.ReceiveAsync(
+          new ArraySegment<byte>(buffer), CancellationToken.None);
+
+      try {
+         while (!receiveResult.CloseStatus.HasValue) {
+            receiveResult = await webSocket.ReceiveAsync(
+                new ArraySegment<byte>(buffer), CancellationToken.None);
+            var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
+            var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
+            var query = rpcReq?.Params?.Query ?? throw new InvalidOperationException(
+               "Failed to parse JsonRpcRequest.Params");
+            var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+            if (exp is null) {
+               Console.WriteLine($"Invalid query expression: {query}.");
+               continue;
+            }
+            var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
+            Disposable.dispose(ref streamSubscription);
+            vs.onChange += OnStreamChanged;
+            streamSubscription = Disposable.Create(() => vs.onChange -= OnStreamChanged);
+         }
+      } catch (Exception ex) {
+         Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
+         return 1;
+      }
+
+      return 0;
+
+      async void OnStreamChanged(object newValue) {
+         if (newValue is not DataPacket dp) {
+            Console.WriteLine(
+               $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
+            return;
+         }
+         Console.WriteLine("Received stream update.");
+         var added_rows_str = dp.addedRows.get_printed_rows(val => val.Length);
+         var msg = $"Received new rows:\n{added_rows_str}";
+         var encoded = Encoding.UTF8.GetBytes(msg);
+         var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
+         await webSocket.SendAsync(
+             payload,
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/WeatherForecastController.cs

@@ -0,0 +1,22 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+using Microsoft.AspNetCore.Mvc;
+
+[ApiController]
+[Route("[controller]")]
+public class WeatherForecastController(ILogger<WeatherForecastController> logger) : ControllerBase {
+   static readonly string[] Summaries = [
+      "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
+   ];
+
+   readonly ILogger<WeatherForecastController> _logger = logger;
+
+   [HttpGet(Name = "GetWeatherForecast")]
+   public IEnumerable<WeatherForecast> Get() {
+      return Enumerable.Range(1, 5).Select(index => new WeatherForecast {
+         Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)),
+         TemperatureC = Random.Shared.Next(-20, 55),
+         Summary = Summaries[Random.Shared.Next(Summaries.Length)]
+      })
+      .ToArray();
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using System.Text.Json.Serialization;
+
+public class DataStreamSubscription {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamUpdate.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using BCG;
+
+class DataStreamUpdate {
+   public List<IBlueRow> addedRows { get; set; } = new(0);
+   public List<IBlueRow> deletedRows { get; set; } = new(0);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerOptions.cs

@@ -0,0 +1,14 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using CommandLine;
+
+class DataStreamWebAppServerOptions {
+   [Option("sparkd")]
+   public string? sparkd { get; set; }
+
+   [Option("user", Default = "bts")]
+   public string? user { get; set; }
+
+   [Option("password", Default = "")]
+   public string? password { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebsocketHandler.cs

@@ -0,0 +1,58 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Net.WebSockets;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Bts.DataStreamWebApp.Server {
+   // public class DataStreamWebSocketHandler {
+   //    readonly WebSocketConnectionManager _connectionManager = new();
+   //    readonly ILogger<DataStreamWebSocketHandler> _logger;
+
+   //    public DataStreamWebSocketHandler(
+   //       WebSocketConnectionManager connectionManager,
+   //       ILogger<DataStreamWebSocketHandler> logger
+   //    ) {
+   //       _connectionManager = connectionManager;
+   //       _logger = logger;
+   //    }
+
+   //    public async Task HandleWebSocket(HttpContext context, WebSocket webSocket) {
+   //       var socketId = _connectionManager.AddSocket(webSocket);
+
+   //       _logger.LogInformation($"WebSocket connection established with ID {socketId}");
+
+   //       while (webSocket.State == WebSocketState.Open) {
+   //          var message = await ReceiveMessageAsync(webSocket);
+   //          if (message != null) {
+   //             _logger.LogInformation($"Received message from ID {socketId}: {message}");
+   //             await BroadcastMessageAsync(message);
+   //          }
+   //       }
+
+   //       _connectionManager.RemoveSocket(socketId);
+   //       _logger.LogInformation($"WebSocket connection closed with ID {socketId}");
+   //    }
+
+   //    async Task<string?> ReceiveMessageAsync(WebSocket webSocket) {
+   //       var buffer = new byte[1024];
+   //       var result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+
+   //       if (result.CloseStatus.HasValue) {
+   //          return null;
+   //       }
+
+   //       return System.Text.Encoding.UTF8.GetString(buffer, 0, result.Count);
+   //    }
+
+   //    async Task BroadcastMessageAsync(string message) {
+   //       foreach (var socket in _connectionManager.GetAllSockets()) {
+   //          if (socket.Value.State == WebSocketState.Open) {
+   //             await socket.Value.SendAsync(System.Text.Encoding.UTF8.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
+   //          }
+   //       }
+   //    }
+   // }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -0,0 +1,159 @@
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.utils;
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+using Bts.DataStreamWebApp.Server;
+using Bts.DataStreamWebApp.Server.Controllers;
+
+using CommandLine;
+
+using mbts.config;
+
+using System.Net.WebSockets;
+using System.Reactive.Disposables;
+using System.Reactive.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+static string fmtReq(HttpRequest req) =>
+   $"{req.Protocol}://{req.Host}/{req.Path}?{req.QueryString}";
+
+static string fmtDataPacket(DataPacket dp) {
+   var sb = new StringBuilder();
+   var rows = dp.addedRows.Cast<IBlueRow>().ToList();
+   var colWidths = dp.schema.columnInfo.Select(c => rows.Max(r => r[c.colNo]?.ToString()?.Length ?? 0)).ToArray();
+   foreach (IBlueRow r in dp.addedRows) {
+      for (int i = 0; i < colWidths.Length; i++) {
+         var str = r[i]?.ToString() ?? string.Empty;
+         if (i > 0) {
+            sb.Append(" | ");
+         }
+         sb.Append(str);
+         var width = colWidths[i];
+         if (str.Length < width) {
+            sb.Append(new string(' ', width - str.Length));
+         }
+      }
+      sb.AppendLine();
+   }
+   return sb.ToString();
+}
+
+var parser = new Parser(ps => {
+   ps.CaseInsensitiveEnumValues = true;
+   ps.HelpWriter = Console.Out;
+});
+
+await parser.ParseArguments<DataStreamWebAppServerOptions>(args).
+   MapResult(run, errors => Task.FromResult(1));
+
+async Task<int> run(DataStreamWebAppServerOptions opts) {
+   BTSNativeMethods.provide_command_line(args, args.Length);
+   Configuration.initialize();
+   var context = ServiceContext.create(opts.sparkd);
+   await context.login(opts.user!, opts.password!);
+   var vsWorkLoop = Observable.Interval(TimeSpan.FromMilliseconds(200)).Subscribe(_ => {
+      ViewStreamDatabase.instance.doWork();
+      MasterWorker.getInstance(false).doWork();
+   });
+   var builder = WebApplication.CreateBuilder(args);
+   builder.Services.AddDirectoryBrowser();
+   var app = builder.Build();
+   var wsOpts = new WebSocketOptions {
+      KeepAliveInterval = TimeSpan.FromMinutes(2)
+   };
+   app.UseWebSockets(wsOpts);
+   app.UseDefaultFiles();
+   app.UseStaticFiles();
+   app.MapFallbackToFile("/index.html");
+
+   app.Use(async (context, next) => {
+      var reqDesc = fmtReq(context.Request);
+      Console.WriteLine($"Webserver received: {reqDesc}");
+      if (context.Request.Path == "/ws") {
+         if (context.WebSockets.IsWebSocketRequest) {
+            Console.WriteLine("Accepting websocket connection...");
+            using var webSocket = await context.WebSockets.AcceptWebSocketAsync();
+            Console.WriteLine("Websocket connection accepted.");
+            await runWebSocketServer(
+               opts.sparkd, opts.user!, opts.password!, webSocket);
+         } else {
+            context.Response.StatusCode = StatusCodes.Status400BadRequest;
+         }
+      } else {
+         Console.WriteLine($"Webserver ignoring req: {reqDesc}");
+         await next(context);
+      }
+   });
+
+   try {
+      await app.RunAsync();
+   } catch (Exception ex) {
+      Console.WriteLine($"Exception on DataStreamWebsocketServer thread: {ex}");
+      return 1;
+   }
+   Spark.Core.Disposables.Disposable.dispose(ref vsWorkLoop);
+   return 0;
+}
+
+async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
+   var buffer = new byte[1024 * 4];
+   var receiveResult = await webSocket.ReceiveAsync(
+       new ArraySegment<byte>(buffer), CancellationToken.None);
+
+   try {
+      while (!receiveResult.CloseStatus.HasValue) {
+         Console.WriteLine("Awaiting websocket message from client");
+         receiveResult = await webSocket.ReceiveAsync(
+             new ArraySegment<byte>(buffer), CancellationToken.None);
+         var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
+         var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
+         var queryArgs = rpcReq!.Params!;
+         // var queryArgs = new DataStreamSubscription { Query = msg };
+         var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
+         if (exp is null) {
+            Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
+            continue;
+         }
+         var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
+         Console.WriteLine("Subscribing to stream...");
+         vs.onChange += onStreamChanged;
+
+         await webSocket.SendAsync(
+             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+      }
+   } catch (Exception ex) {
+      Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
+      return 1;
+   }
+
+   return 0;
+
+   async void onStreamChanged(object newValue) {
+      if (newValue is not DataPacket dp) {
+         Console.WriteLine(
+            $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
+         return;
+      }
+      Console.WriteLine("Received stream update.");
+      var sb = new StringBuilder();
+      var updateDesc = fmtDataPacket(dp);
+      var msg = $"Received new rows:\n{updateDesc}";
+      // var payload = new ReadOnlyMemory<byte>([.. msg]);
+      var encoded = Encoding.UTF8.GetBytes(msg);
+      var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
+      await webSocket.SendAsync(
+          payload,
+          receiveResult.MessageType,
+          receiveResult.EndOfMessage,
+          CancellationToken.None);
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -0,0 +1,48 @@
+{
+  "profiles": {
+    "default": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5400"
+    },
+    "http": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "http://localhost:5268"
+    },
+    "https": {
+      "commandName": "Project",
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "https://localhost:7128;http://localhost:5268"
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      }
+    }
+  },
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:2083",
+      "sslPort": 44383
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WeatherForecast.cs

@@ -0,0 +1,12 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class WeatherForecast
+{
+   public DateOnly Date { get; set; }
+
+   public int TemperatureC { get; set; }
+
+   public int TemperatureF => 32 + (int)(TemperatureC / 0.5556);
+
+   public string? Summary { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WebsocketConnectionManager.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Concurrent;
+using System.Net.WebSockets;
+
+namespace Bts.DataStreamWebApp.Server {
+   // public class WebSocketConnectionManager {
+   //    private readonly ConcurrentDictionary<Guid, WebSocket> _sockets = new ConcurrentDictionary<Guid, WebSocket>();
+
+   //    public Guid AddSocket(WebSocket socket) {
+   //       var socketId = Guid.NewGuid();
+   //       _sockets.TryAdd(socketId, socket);
+   //       return socketId;
+   //    }
+
+   //    public WebSocket? GetSocket(Guid socketId) {
+   //       _sockets.TryGetValue(socketId, out var socket);
+   //       return socket;
+   //    }
+
+   //    public ConcurrentDictionary<Guid, WebSocket>.ValueCollection GetAllSockets() {
+   //       return _sockets.Values;
+   //    }
+
+   //    public Guid? GetSocketId(WebSocket socket) {
+   //       foreach (var (key, value) in _sockets) {
+   //          if (value == socket) {
+   //             return key;
+   //          }
+   //       }
+   //       return null;
+   //    }
+
+   //    public void RemoveSocket(Guid socketId) {
+   //       _sockets.TryRemove(socketId, out _);
+   //    }
+   // }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/DataStream/Misc/PandasScriptStream.cs

@@ -1,29 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-
-using System;
-using System.IO;
-
-public class PandasScriptStream : ViewStream {
-   public PandasScriptStream(ViewStream vs, string file)
-      : this(vs, file, null) { }
-
-   public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
-      : this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
-
-   public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
-      mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
-
-      string readFile() {
-         try {
-            return File.ReadAllText(file);
-         } catch (Exception ex) {
-            Log.Gen.error($"PandasScriptStream error reading file: {ex}");
-            throw;
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,130 +1,62 @@
-namespace Bts.DataStream;
-
-using BCG;
-using BCG.Extensions;
-using BCG.Utilities.General;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Runtime.CompilerServices;
-using System.Text;
-
-public class PandasStream : ViewStream, IWorker {
-   readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
-   readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
-   readonly string pythonExe;
-   readonly ViewStreamDatabase vsd;
-   readonly CSVSaverStream inputCsvStream;
-   readonly Lazy<string> pythonCode;
-   IDictionary<int, IBlueRow>? lastRowMap = null;
-   int inputVersion = 0;
-   int outputVersion = 0;
-   DateTime lastFireTime = DateTime.MinValue;
-   TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
-   IDisposable? dispose;
-
-   public PandasStream(ViewStream vs, string dataFrameExpression)
-      : this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
-
-   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
-      pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
-      this.pythonExe = pythonExe;
-      this.vsd = vsd;
-      // This should fire an update when it writes to the output file
-      inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
-      pythonCode = new(getPythonCode);
-
-      string getPythonCode() {
-         var imports =
-@"import numpy as np
-import pandas as pd
-import sys
-";
-         var bd = new StringBuilder(imports);
-         bd.AppendLine($"df=pd.read_csv('{inputFile}')");
-         bd.AppendLine(dataFrameExpression);
-         bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
-         return bd.ToString();
-      }
-   }
-
-   protected override void myOnFirstSubscriber() {
-      vsd.addWorker(this);
-      inputCsvStream.onChange += onInputChanged;
-      this.dispose = Disposable.Create(dispose);
-
-      void onInputChanged(object newValue) =>
-         inputVersion++;
-
-      void dispose() {
-         inputCsvStream.onChange -= onInputChanged;
-         vsd.removeWorker(this);
-         deleteFile(inputFile);
-         deleteFile(outputFile);
-      }
-
-      void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
-         if (!File.Exists(file)) {
-            return;
-         }
-         try {
-            File.Delete(file);
-         } catch (Exception ex) {
-            Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose(ref dispose);
-
-   static void executePythonCode(string pythonCode, string? pythonExe = null) {
-      pythonExe ??= "python";
-      var start = new ProcessStartInfo {
-         FileName = pythonExe,
-         Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
-         UseShellExecute = false,
-         RedirectStandardOutput = true
-      };
-      using var process = Process.Start(start)
-         ?? throw new InvalidOperationException("Failed to launch python");
-      process.WaitForExit();
-   }
-
-   void IWorker.doWork() {
-      if (outputVersion == inputVersion) {
-         return;
-      }
-      var now = DateTime.Now;
-      var elapsedSinceLastFire = now - lastFireTime;
-      if (elapsedSinceLastFire < fireThrottle) {
-         return;
-      }
-      outputVersion = inputVersion;
-      executePythonCode(pythonCode.Value, pythonExe);
-      var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
-      var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
-      var comp = BlueRowEqualityComparer.Instance;
-      var deletedRows = new BlueTable(schema);
-      var addedRows = new BlueTable(schema);
-      if (lastRowMap is not null) {
-         deletedRows.SetItems(lastRowMap.Where(oldRow =>
-            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
-         addedRows.SetItems(rowMap.Where(newRow =>
-            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
-      }
-      lastRowMap = rowMap;
-      lastFireTime = DateTime.Now;
-      myFireEvent(new DataPacket(rows, addedRows, deletedRows));
-   }
-}
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+//     public class PandasStream : ViewStream, IWorker {
+//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+//         }
+
+//         public PandasStream(ViewStream vs, string pandas_code) :
+//         this(vs, ViewStreamDatabase.instance, pandas_code) {
+//         }
+
+
+//         protected override void myOnFirstSubscriber() {
+
+//             // string python_code = @"
+//             //    import winsound
+//             //    frequency = 2500  # Set frequency (in Hz)
+//             //    duration = 1000   # Set duration (in milliseconds)
+//             //    winsound.Beep(frequency, duration)
+//             // ";
+//             string python_code = @"
+//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
+//             with open(file_path, \"w\") as f:
+//                 f.write(\"Hello, world!\n\")
+//             ";
+
+//             execute_python_code(python_code);
+//         }
+
+//         static void execute_python_code(string python_code) {
+//             ProcessStartInfo start = new ProcessStartInfo();
+//             start.FileName = "python"; // Python executable
+//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+//             start.UseShellExecute = false;
+//             start.RedirectStandardOutput = true;
+
+//             using (Process process = Process.Start(start)) {
+//                 using (System.IO.StreamReader reader = process.StandardOutput) {
+//                     string result = reader.ReadToEnd();
+//                     Console.Write(result);
+//                 }
+//             }
+//         }
+
+//         protected override void myOnLastSubscriber() {
+//         }
+
+//         public void doWork() {
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -14,11 +14,8 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
       this.source = source;
 
    public void Dispose() {
-      GC.SuppressFinalize(this);
-      if (subscription is not null) {
-         subscription.Dispose();
-         subscription = null;
-      }
+      subscription?.Dispose();
+      subscription = null;
    }
 
    protected override void myOnFirstSubscriber() {

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -0,0 +1,32 @@
+using BCG;
+
+namespace BCG
+{
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("add_to_cloud")]
+   public class AddToCloudStream : ViewStream
+   {
+      private readonly string _name, _stream_exp;
+      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
+      public AddToCloudStream(string name, params string[] lines)
+      : this(name, 0, lines)
+      {
+      }
+
+      public AddToCloudStream(string name, int persist, params string[] lines)
+      {
+         _name = name;
+         _stream_exp = getExpression(lines);
+         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
+         mySetRealSource(new TableStream("cloud_streams"));
+      }
+
+      static string getExpression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,22 +1,31 @@
+using BCG;
+
 using System;
 using System.Collections.Generic;
 
+using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
+using System.Collections.Generic;
 using BCG.Utilities.General;
-using bts;
 
-namespace BCG {
+namespace BCG
+{
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IWorker {
-      public class StreamPubUnit {
+   public class CloudStreamManager : IWorker
+   {
+      public class StreamPubUnit
+      {
          public string name;
          public string stream_expression;
          public ViewStream stream;
          public bool persist;
          public ViewPublisher view_publisher;
 
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
+         {
             name = name_;
             stream_expression = stream_expression_;
             stream = stream_;
@@ -24,7 +33,7 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream? build_stream(string stream_expression)
+      static ViewStream build_stream(string stream_expression)
       {
          try
          {
@@ -32,7 +41,9 @@ namespace BCG {
             var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
@@ -41,37 +52,34 @@ namespace BCG {
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       private ViewStream _stream = new TableStream("cloud_streams");
       private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
-
-      public RemoteQueryManager() {
+      public CloudStreamManager()
+      {
          _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
+
       }
       private void _on_str(object obj)
       {
-         if (obj is not DataPacket dp) {
-            return;
-         }
+         DataPacket dp = obj as DataPacket;
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
             bool persist = r.get<bool>("persist");
             string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name)) {
-               _spawn_remote_stream(name, stream_exp, persist);
+            if (!_stream_pub_map.ContainsKey(name))
+            {
+               _assign(name, stream_exp, persist);
             }
          }
       }
 
-      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
+      private void _assign(string name, string stream_expression, bool persist)
+      {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
-         if (persist) {
+         if (persist)
+         {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -79,12 +87,14 @@ namespace BCG {
          _stream_pub_map.Add(name, u);
       }
 
-      private void _expose_to_ws(string name, ViewStream stream) {
+      private void _expose_to_ws(string name, ViewStream stream)
+      {
          // TODO: john
       }
 
-      public void doWork() {
+      public void doWork()
+      {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -1,48 +0,0 @@
-using BCG;
-
-using bts;
-
-using System;
-
-namespace BCG {
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("remoteQuery")]
-   public class RemoteQueryStream : ViewStream {
-      private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = new PublisherCore();
-      private ViewStream _source_str;
-
-      public RemoteQueryStream(params string[] lines) {
-         _stream_exp = get_expression(lines);
-         _name = Guid.NewGuid().ToString();
-         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
-         //_name.Replace("-", "_");
-
-      }
-
-      protected override void myOnFirstSubscriber() {
-         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
-         byte[] rem_req_bytes = rem_req.serialize();
-         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
-         _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
-      }
-
-      protected override void myOnLastSubscriber() {
-         //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
-      }
-
-      static string get_expression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-
-      private void _handle_data(object o) {
-         myFireEvent(o);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -1,31 +0,0 @@
-namespace bts {
-   using System.IO;
-   using BCG;
-
-   using System.IO;
-   using System.Threading.Tasks.Dataflow;
-
-   public class RemoteRequest {
-      public readonly string stream_expression;
-      public readonly string published_table_name;
-      public RemoteRequest(string stream_expression_, string published_table_name_) {
-         stream_expression = stream_expression_;
-         published_table_name = published_table_name_;
-      }
-      public byte[] serialize() {
-         var str = new MemoryStream();
-         var bw = new BinaryWriter(str);
-         bw.write_string_cpp(stream_expression);
-         bw.write_string_cpp(published_table_name);
-         return str.ToArray();
-      }
-
-      public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
-         BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
-         string stream_expression_ = br.read_string_cpp();
-         string published_table_name_ = br.read_string_cpp();
-         return new RemoteRequest(stream_expression_, published_table_name_);
-
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -1,54 +1,59 @@
-// using BCG;
+using BCG;
 
-// using bts;
+using System.Linq;
 
-// using System.Linq;
+namespace BCG
+{
+   public class StreamOffloadRequestor
+   {
+      public class OffloadOp
+      {
+         string name;
+         string stream_exp;
+      }
+      private IBlueTableSchema _schema;
+      private IBlueRowCollection _rows;
+      private PubTableBroker _broker = new PubTableBroker();
+      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+      public StreamOffloadRequestor()
+      {
+         _schema = new BlueTableSchema("cloud_streams",
+                                   new ColumnInfo("id", 0, typeof(int)),
+                                   new ColumnInfo("name", 1, typeof(string)),
+                                   new ColumnInfo("persist", 2, typeof(bool)),
+                                   new ColumnInfo("stream_exp", 3, typeof(string)));
+         _existing_offloads_vs.onChange += _on_str;
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         _rows = dp.newRows;
+         // this is needed so the broker knows the state of the table
+         _broker.createTable(_rows);
+      }
+      public void add_to_cloud(string name, string stream_exp, bool persist)
+      {
+         // check if it exists
+         if (_rows == null)
+         {
+            BlueTable table = new BlueTable(_schema);
+            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+            table.addRow(nr);
+            _broker.createTable(table);
+            // create table  
+         }
+         else
+         {
+            if (exists(name))
+               return;
+            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+            _broker.addRow(nr);
+         }
+      }
 
-// namespace BCG {
-//    // used on the client
-//    public class StreamOffloadRequestor {
-//       public class OffloadOp {
-//          string name;
-//          string stream_exp;
-//       }
-//       private IBlueTableSchema _schema;
-//       private IBlueRowCollection _rows;
-//       private PubTableBroker _broker = new PubTableBroker();
-//       ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-//       public StreamOffloadRequestor() {
-//          _schema = new BlueTableSchema("cloud_streams",
-//                                    new ColumnInfo("id", 0, typeof(int)),
-//                                    new ColumnInfo("name", 1, typeof(string)),
-//                                    new ColumnInfo("persist", 2, typeof(bool)),
-//                                    new ColumnInfo("stream_exp", 3, typeof(string)));
-//          _existing_offloads_vs.onChange += _on_str;
-//       }
-//       private void _on_str(object obj) {
-//          DataPacket dp = obj as DataPacket;
-//          _rows = dp.newRows;
-//          // this is needed so the broker knows the state of the table
-//          _broker.create_table(_rows);
-//       }
-//       public void add_to_cloud(string name, string stream_exp, bool persist) {
-//          // check if it exists
-//          if (_rows == null) {
-//             BlueTable table = new BlueTable(_schema);
-//             IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-//             table.addRow(nr);
-//             _broker.create_table(table);
-//             // create table  
-//          } else {
-//             if (exists(name))
-//                return;
-//             IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-//             // TODO: use make_batch_changes instead
-//             Log.assert(false, "Fix addRow");
-//             //_broker.addRow(nr);
-//          }
-//       }
-
-//       bool exists(string name) {
-//          return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-//       }
-//    }
-// }
+      bool exists(string name)
+      {
+         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs

@@ -1,14 +1,17 @@
 using System;
 using System.Collections;
 
-namespace BCG {
-   internal class AddColumnsRow : IBlueRow {
+namespace BCG
+{
+   internal class AddColumnsRow : IBlueRow
+   {
       private IBlueTableSchema mySchema;
       private IBlueRow myRow;
       //private object[] myCachedValues;
 
       #region Constructors
-      public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
+      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
+      {
          mySchema = schema;
          myRow = row;
          //myCachedValues = new object[schema.columnNames.Length];
@@ -16,78 +19,85 @@ namespace BCG {
       #endregion
 
       #region IBlueRow members
-      public IBlueTableSchema schema { get { return mySchema; } }
+      public IBlueTableSchema schema{get{return mySchema;}}
       #endregion
 
       #region ICollection members
-      public int Count { get { return mySchema.columnNames.Length; } }
+      public int Count {get{return mySchema.columnNames.Length;}}
 
-      public bool IsSynchronized { get { return false; } }
+      public bool IsSynchronized {get{return false;}}
 
-      public object SyncRoot { get { return this; } }
+      public object SyncRoot {get{return this;}}
 
-      public void CopyTo(Array array, int index) {
+      public void CopyTo (Array array, int index)  
+      {
          int i = index;
          foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
+            array.SetValue (o, i++);
       }
       #endregion
 
       #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
+      public bool IsFixedSize {get{return true;}}
 
-      public bool IsReadOnly { get { return true; } }
+      public bool IsReadOnly {get{return true;}}
 
       /// <summary>
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public object this[object key] {
-         get {
+      public object this [object key]
+      {
+         get
+         {
             if (key is string)
                return this[(string)key];
             return this[(int)key];
          }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+         set {throw new NotSupportedException ("The IBlueRow is readonly");}
       }
 
-      public object this[int index] {
-         get {
+      public object this[int index]
+      {
+         get
+         {
             //if (myCachedValues[index] == null)
             //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
             //return myCachedValues[index];
-            return mySchema[index].isConstant ? DBNull.Value : myRow[index];
+            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
          }
       }
 
-      public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
+      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}
 
-      public ICollection Keys { get { return mySchema.columnNames; } }
+      public ICollection Keys{get{return mySchema.columnNames;}}
 
-      public ICollection Values {
-         get {
+      public ICollection Values
+      {
+         get
+         {
             object[] retVal = new object[mySchema.columnNames.Length];
-            CopyTo(retVal, 0);
+            CopyTo (retVal, 0);
             return retVal;
          }
       }
 
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
 
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
 
-      public bool Contains(object key) { return mySchema[(string)key] != null; }
+      public bool Contains (object key){return mySchema[(string)key] != null;}
 
       /// <summary>
       /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
       /// iterate through the values, typecast this into an IEnumerable
       /// </summary>
       /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
+      
+      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
 
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
       #endregion
 
    }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,34 +1,6 @@
-using System;
-
-namespace BCG {
-   public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_exp(string exp) {
-         string[] toks = exp.Split(':');
-         string col_name = toks[0];
-         string col_value = toks[1];
-         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return Tuple.Create(col_name, tup.Item1, tup.Item2);
-      }
-
-      private Tuple<string, Type, object> _new_col;
-      private IBlueTableSchema _out_schema;
-
-      public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
-         mySetRealSource(rs);
-      }
-
-      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null)
-            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
-         return _out_schema;
-      }
-
-      private IBlueRow _map(IBlueRow r) {
-         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         rv.setValue(_new_col.Item1, _new_col.Item3);
-         return rv;
-      }
-   }
-}
+namespace BCG;
+
+public class AddColumnsStream : MapStream {
+   public AddColumnsStream(ViewStream vs, string columnDef)
+      : base(vs, new ColumnAdder(columnDef)) { }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,24 +1,48 @@
-using BCG;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
 
-namespace bts {
+using System.Runtime.Serialization.Formatters.Binary;
+
+namespace BCG
+{
    [DataStreamUserRespresentation("loader")]
-   public class LoaderStream : ViewStream {
+   public class LoaderStream : ViewStream, ISideEffectStream
+   {
+      public class Exception : System.Exception
+      {public Exception(string msg) : base(msg) { }}
 
-      private string _filename;
-      private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
-
-
-      public LoaderStream(string filename) {
-         _filename = filename;
+      public LoaderStream(string fileName, int secondsBetweenPolls)
+      {
+         mySetRealSource(
+            new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+               ViewStreamDatabase.instance));
       }
 
-      protected override void myOnFirstSubscriber() {
-         byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
-         StreamIOMessage msg = _deser.deserialize(ser_msg);
-         Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
-         IBlueRowCollection rows = msg.create_message.rows;
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
+      public LoaderStream(string fileName)
+      {
+        if (fileName == null)
+           throw new LoaderStream.Exception("File name cannot be null");
+
+        for (int i = 0; i < 5; i++)
+           try
+           {
+              using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+              {
+#pragma warning disable SYSLIB0011
+                  BinaryFormatter bf = new BinaryFormatter();
+                  BlueTable bt = (BlueTable)bf.Deserialize(fs);
+#pragma warning restore SYSLIB0011
+                  mySetRealSource(new BlueRowCollectionStaticStream(bt));
+              }
+              break;
+           }
+           catch (IOException)
+           {
+              System.Threading.Thread.Sleep(50);
+           }
+
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,7 +5,6 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
 
 namespace BCG {
    // Writes all the udpates to a stream in a file.
@@ -25,7 +24,7 @@ namespace BCG {
       private ViewStream _source_stream;
       private string _token;
       private FileStream _out_fs;
-      private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
+      private IBlueDataSerializer _serializer;
       private string _file_name = String.Empty;
       private long _rollover_size;
       private readonly string _directory_path;
@@ -95,12 +94,13 @@ namespace BCG {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
          bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalStreamIOSerializer();
+         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
          if (_write_create) {
             // new files always starts with create
-            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
+            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
          } else
-            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
+            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
+
          BinaryWriter bw = new BinaryWriter(_out_fs);
          long utc_epoch = epoch_now();
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +1,68 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
 
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
+namespace BCG
+{
+   public class RecorderStream : ViewStream
+   {
+      private ViewStream mySource;
+      private DateTime myStartTime;
+      private FileStream myOutFile;
+      private string myFileName;
+      private IBlueDataSerializer mySerializer;
+            
+      public RecorderStream(ViewStream src, string recordFile)
+      {
+         mySource = src;
+         myFileName = recordFile;
 
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
+         myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+      }
 
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
+      protected override void myOnFirstSubscriber()
+      {
+         myStartTime = DateTime.Now;         
+         mySource.onChange += new ChangeDelegate(mySource_onChange);
+         myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+      }
+      
+      protected override void myOnLastSubscriber()
+      {
+         mySource.onChange -= mySource_onChange;
+         myOutFile.Close();
+      }
 
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
+      void mySource_onChange(object newValue)
+      {
+         DataPacket dp = (DataPacket)newValue;         
+         
+         BlueTable toWrite = new BlueTable(dp.schema);
 
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
+         if (mySerializer == null)
+         {
+            mySerializer = new UniversalSerializer(dp.schema);
+         }
+         foreach (IBlueRow r in dp.newRows)
+            toWrite.addRow(r);
+         
+         byte[] result;
 
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
+         if (myLastValue == null)
+            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+         else
+            mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
 
-//          BlueTable toWrite = new BlueTable(dp.schema);
+         BinaryWriter bw = new BinaryWriter(myOutFile);
+         //write down what time this happened
+         bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+         bw.Write(result.Length);
+         bw.Write(result);         
+         myOutFile.Flush();
 
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }
+         myFireEvent(dp);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -1,37 +1,64 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
-using BCG;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
 
-namespace bts {
-   // this saves the total state in the filr provided in the cons
-   [DataStreamUserRespresentation("saver")]
-   public class SaverStream : ViewStream {
-      private ViewStream _stream;
-      private string _filename;
-      private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
+// using System.Runtime.Serialization.Formatters.Binary;
 
-      public SaverStream(ViewStream vs, string filename) {
-         _stream = vs;
-         _filename = filename;
-      }
+// namespace BCG
+// {
+//    [DataStreamUserRespresentation("saver")]
+//    public class SaverStream : ViewStream, ISideEffectStream
+//    {
+//       private ViewStream myStream;
+//       private string myFileName;
 
-      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
+//       public class Exception : System.Exception
+//       {
+//          public Exception(string msg) : base(msg) { }
+//       }
 
-      void _handle_changes(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         _save(dp);
-         myFireEvent(dp_obj);
-      }
+//       public SaverStream(ViewStream vs, string fileName)
+//       {
+//          if (fileName == null)
+//             throw new SaverStream.Exception("File name cannot be null");
+//          myStream = vs;
+//          myFileName = fileName;
 
-      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
+//       }
 
-      protected void _save(DataPacket dp) {
-         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
-         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
-         StreamSerializationUtils.write_to_file(str_bytes, _filename);
+//       protected override void myOnFirstSubscriber()
+//       { myStream.onChange += new ChangeDelegate(myHandleChanges); }
 
-      }
-   }
-}
+//       void myHandleChanges(object newValue)
+//       {
+//          DataPacket dp = newValue as DataPacket;
+//          mySave(dp);
+//          myFireEvent(newValue);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
+
+//       protected void mySave(DataPacket dp)
+//       {
+//          for (int i = 0; i < 5 ; i++)
+//             try
+//             {
+//                using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
+//                {
+//                   BinaryFormatter bf = new BinaryFormatter();
+//                   BlueTable t = new BlueTable(dp.schema);
+//                   foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
+//                      t.addRow(new BlueRow(r));
+//                   bf.Serialize(fs, t);
+//                }
+//                break;
+//             }
+//             catch (IOException)
+//             {
+//                System.Threading.Thread.Sleep(5);
+//             }
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -1,12 +1,12 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Runtime.Serialization.Formatters.Binary;
+using BCG.Utilities.General;
+using bts.collections;
+using System.Reflection.Metadata;
 
 
 namespace BCG {
@@ -15,115 +15,115 @@ namespace BCG {
          FirstOfDay,
          LastOfDay
       };
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       public DateTime _snapshot_time;
-//       public string _snapshot_file;
-//       public IEnumerable<PlaybackPacket> _packets;
-//       public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-//          DateTime dt;
-//          var parsed = DateTime.TryParse(snapshot_date, out dt);
-//          dt = dt.Date;
-//          bts.Log.assert(parsed);
-//          SpecificSnapshot specific_snapshot_enum;
-//          parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-//          bts.Log.assert(parsed);
-//          string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-//          ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-//          mySetRealSource(rs);
-//       }
+      private BlueTable _current_table;
+      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+      public DateTime _snapshot_time;
+      public string _snapshot_file;
+      public IEnumerable<PlaybackPacket> _packets;
+      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+         DateTime dt;
+         var parsed = DateTime.TryParse(snapshot_date, out dt);
+         dt = dt.Date;
+         bts.Log.assert(parsed);
+         SpecificSnapshot specific_snapshot_enum;
+         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+         bts.Log.assert(parsed);
+         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+         mySetRealSource(rs);
+      }
 
-//       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-//          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-//       }
+      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+         string token_directory_name = $"{directory_name}\\{token}";
+         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+      }
 
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-//          _packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
+      protected override void myOnFirstSubscriber() {
+         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+         _packets = StreamPersistorUtils.load_packets_from_file(fs);
+         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+         if (snapshot_packet != null) {
+            foreach (PlaybackPacket p in _packets) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+               for (int i = 0; i < ops.Length; i++) {
+                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+                  byte op = ops[i];
+                  if (op == StreamOpCode.create) {
+                     IBlueRowCollection full_table = rows_args[0];
+                     _current_table = new BlueTable(full_table);
+                  } else if (op == StreamOpCode.multipleRemoveAdd) {
+                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
+                  } else {
+                     bts.Log.assert(false, $"Oprator: {op} is not expected");
+                  }
+                  if (p.idx == snapshot_packet.idx) {
+                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+                     break;
+                  }
+               }
+            }
+         }
+      }
 
-//       private void _absorb_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//             } else {
-//                bts.Log.assert(false, $"Operator: {op} is not expected");
-//             }
-//          }
-//       }
+      private void _absorb_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+            } else {
+               bts.Log.assert(false, $"Operator: {op} is not expected");
+            }
+         }
+      }
 
-//       protected override void myOnLastSubscriber() {
-//       }
+      protected override void myOnLastSubscriber() {
+      }
 
-//       // public event SnapshotStream.Progress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          return null;
-//       }
+      // public event SnapshotStream.Progress onProgress;
+      // public event Action onFileExhausted;
+      private PlaybackPacket _locate_packet_to_replay() {
+         foreach (var p in _packets) {
+            // returns the first non replayed packet
+            if (!p.replayed)
+               return p;
+         }
+         return null;
+      }
 
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
+      private void _fire_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+               //_current_table.addRows(full_table);
+               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+                                          (myLastValue as DataPacket).newRows));
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+               myFireEvent(new DataPacket(_current_table, adds, dels));
+            } else {
+               bts.Log.assert(false, $"Oprator: {op} is not expected");
+            }
 
-//          }
-//       }
+         }
+      }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,10 +1,16 @@
+using BCG;
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using System.Text.RegularExpressions;
+
 using Bts.DataStream;
 
+using mbts;
+using mbts.spark;
 using mbts.config;
 
-using System;
-using System.Text.RegularExpressions;
-
 namespace BCG.StreamWorld.Addition
 {
    [DataStreamUserRespresentation (false)]
@@ -145,6 +151,7 @@ namespace BCG
       private string myFilterString;
       private BCG.StreamWorld.Addition.FilterStream myFilterStream;
       private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
+      private HighPerformanceTimer _timer = new HighPerformanceTimer();
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -219,6 +226,7 @@ namespace BCG
 
       private void myHandleCallbacks(object newValue)
       {
+         _timer.start();
          DataPacket retVal;
          DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
@@ -247,6 +255,7 @@ namespace BCG
                if (myFilter.filter(r))
                   myLastFiredRows.addRow(r);
          }
+         _timer.stop();
          myFireEvent(retVal);
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -1,66 +1,37 @@
 using System;
-using System.Text.RegularExpressions;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
 using BCG.Utilities.General;
 
-namespace BCG {
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
-
-      public static TimeSpan parse_duration(string input) {
-         int days = 0, hours = 0, minutes = 0, seconds = 0;
-
-         // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
-         Regex regex = new Regex(@"(\d+)([dhms])");
-         MatchCollection matches = regex.Matches(input);
-
-         foreach (Match match in matches) {
-            int value = int.Parse(match.Groups[1].Value);
-            string unit = match.Groups[2].Value;
-
-            switch (unit) {
-               case "d":
-                  days = value;
-                  break;
-               case "h":
-                  hours = value;
-                  break;
-               case "m":
-                  minutes = value;
-                  break;
-               case "s":
-                  seconds = value;
-                  break;
-            }
-         }
-
-         return new TimeSpan(days, hours, minutes, seconds);
-      }
-
+namespace BCG
+{
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
+   {
       #region constructors
-
-      public TimeBasedFilter(string col_name, string row_time_to_live)
-      : this(col_name, parse_duration(row_time_to_live)) {
-
-      }
-
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
+      {
       }
 
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
+         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
+      {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+         : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
+      {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
-         myVsd = vsd;
-         _time_col = rowTimeColumnName;
-         _ttl = rowTimeToLive;
-         myFilterFrequency = filterFrequency;
-         myNextFilter = DateTime.MinValue;
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
+      {
+         myVsd               = vsd;
+         myRowTimeColumnName = rowTimeColumnName;
+         myRowTimeToLive     = rowTimeToLive;
+         myFilterFrequency   = filterFrequency;
+         myNextFilter        = DateTime.MinValue;
 
          // add worker
          myVsd.addWorker(this);
@@ -70,25 +41,32 @@ namespace BCG {
       public event FilterChangeDelegate onFilterChange;
 
       #region IStreamFilter
-      public bool filter(IBlueRow row) {
-         DateTime now = DateTime.Now;
+      public bool filter(IBlueRow row)
+      {
+         DateTime now       = DateTime.Now;
+         DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
 
-         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
-         if (now > row_time) {
-            TimeSpan span = now - row_time;
-            if (span >= _ttl) {
+         if (now > timeStamp)
+         {
+            TimeSpan span = now - timeStamp;
+
+            if (span >= myRowTimeToLive)
+            {
                return (false);
             }
          }
+
          return (true);
       }
       #endregion
 
       #region IWorker
-      public void doWork() {
+      public void doWork()
+      {
          DateTime now = DateTime.Now;
 
-         if (now > myNextFilter) {
+         if (now > myNextFilter)
+         {
             // fire event
             onFilterChange();
 
@@ -98,7 +76,8 @@ namespace BCG {
       #endregion
 
       #region IDisposable
-      public void Dispose() {
+      public void Dispose()
+      {
          // remove worker
          myVsd.removeWorker(this);
       }
@@ -106,20 +85,10 @@ namespace BCG {
 
       #region state
       private ViewStreamDatabase myVsd;
-      private string _time_col;
-      private TimeSpan _ttl;
-      private TimeSpan myFilterFrequency;
-      private DateTime myNextFilter;
+      private string             myRowTimeColumnName;
+      private TimeSpan           myRowTimeToLive;
+      private TimeSpan           myFilterFrequency;
+      private DateTime           myNextFilter;
       #endregion
    }
-   [DataStreamUserRespresentation("timefilter")]
-   public class TimeBasedFilterStream : ViewStream {
-      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
-      :
-      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
-      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
-         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
-         mySetRealSource(new FilterStream(base_stream, filter));
-      }
-   }
 }