temp stuff working

It only needs to connect to sparkd if it's publishing market data

.devcontainer/Dockerfile

@@ -0,0 +1,122 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Note installations that take a while (more than 10s)
+
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
+
+# Python: ~30s
+RUN apt-get install -y \
+   python3-pip \
+   python3-venv \
+   && python3 -m pip install --upgrade pip
+
+# GCC
+ARG gcc_major=13
+RUN \
+   add-apt-repository ppa:ubuntu-toolchain-r/test \
+   && apt-get update \
+   && apt-get install -y g++-$gcc_major
+
+# Clang
+ARG clang_major=17
+RUN \
+   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+   | tee -a "/etc/apt/sources.list.d/llvm.list" \
+   && apt-get update \
+   && apt-get install -y \
+   clang-$clang_major \
+   clang-format-$clang_major \
+   clangd-$clang_major
+
+# CMake
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
+
+# Dotnet
+RUN apt-get update && apt-get install -y dotnet7
+
+# Node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+   | tee /etc/apt/sources.list.d/nodesource.list \
+   && apt-get update \
+   && apt-get install -y nodejs
+
+# Emumba (deps)
+RUN apt-get update && apt-get -y install rapidjson-dev
+# Emumba
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
+
+# Pcap
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+
+# Set up default executables (update-alternatives)
+RUN \
+   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+
+# Add compiler symlinks for ccache 
+RUN \
+   ln -s /usr/bin/ccache /usr/local/bin/clang; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+   ln -s /usr/bin/ccache /usr/local/bin/g++
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+   groupadd -g "$docker_uid" "$docker_user" \
+   && useradd \
+   -m \
+   -u "$docker_uid" \
+   -g "$docker_uid" \
+   -G sudo \
+   --password "$(openssl passwd -1 "$docker_pass")" \
+   "$docker_user"

.devcontainer/devcontainer.json

@@ -0,0 +1,48 @@
+{
+  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  // "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+  ],
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
+      "extensions": [
+        "eamodio.gitlens",
+        "llvm-vs-code-extensions.vscode-clangd",
+        "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools"
+      ]
+    }
+  }
+}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.editorconfig

@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles
 
-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower
 
-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion

.gitattributes

@@ -0,0 +1,2 @@
+*.cmd eol=crlf
+*.sh eol=lf

.gitignore

@@ -8,7 +8,9 @@ cmake-build-debug/
 .idea/
 .dir-locals.el
 *.pyc
-**.vscode
+**.vscode/*
+!.vscode/launch.json
+!.vscode/tasks.json
 .vs
 obj/
 *.csproj.user
@@ -54,6 +56,6 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-**/.devcontainer
+.devcontainer/
 *.Generated.cs
 *.lst

.vscode/launch.json

@@ -0,0 +1,154 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "${workspaceFolder}/build/sparkd-debug-gcc/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "${workspaceFolder}/build/sparkd-debug-gcc/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    
+    {
+      "name": "DataStreamPublisher",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DataStreamQueryAnalyzer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
+        "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "PubSubServer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Parameterized)",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build Spark (Debug)",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "args": [
+          "--sparkd",
+          "${input:sparkConfigServer}"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sparkd",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
+      "program": "",
+      "linux": {
+        "program": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "build/sparkd-debug-clang/server",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    }
+  ],
+  "inputs": [
+    {
+      "id": "dataStreamPublisherConfig",
+      "type": "pickString",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "sparkConfigServer",
+      "type": "promptString",
+      "description": "Config Server URL",
+      "default": "http://localhost:8000/config",
+    },
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,773 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Build Api Server",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Api Server (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Api Server (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--preset",
+          "${input:sparkdCmakePreset}",
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd -j8",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--preset",
+          "${input:sparkdCmakePreset}",
+          "-j",
+          "8"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "${input:sparkdCompiler}"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug | Gcc)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "gcc"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug | Clang)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "clang"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "${input:sparkdCompiler}"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release | Gcc)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "gcc"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release | Clang)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "clang"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Edit DataStream Config",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "editor",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/conf/spark.conf"
+        ],
+      },
+      "windows": {
+        "command": "vim",
+        "args": [
+          "${env:APPDATA}/bts/DataStream/conf/spark.conf"
+        ],
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "Edit Spark Config",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "vim",
+        "args": [
+          "${env:APPDATA}/bts/conf/spark.conf"
+        ],
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "dedicated",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Spark",
+      "dependsOn": "Build Spark",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Release)",
+      "dependsOn": "Build Spark (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark Config Query",
+      "dependsOn": "Build Spark",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "Spark.Console",
+      },
+      "windows": {
+        "command": "Spark.Console.exe",
+      },
+      "args": [
+        "config",
+        "--query",
+        "${input:sparkConfigQuery}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd Config Query",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "config",
+        "--query",
+        "${input:sparkdConfigQuery}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "DataStreamQueryAnalyzer.exe",
+          "'Starting DataStreamQueryAnalyzer'"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+  ],
+  "inputs": [
+    {
+      "id": "buildConfiguration",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "Debug",
+        "Release",
+      ],
+      "default": "Debug",
+    },
+    {
+      "id": "sparkdCmakePreset",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "sparkd-debug-clang",
+        "sparkd-debug-gcc",
+        "sparkd-release-clang",
+        "sparkd-release-gcc",
+      ],
+      "default": "sparkd-debug-clang",
+    },
+    {
+      "id": "sparkdCompiler",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "clang",
+        "gcc",
+      ],
+      "default": "sparkd-debug-clang",
+    },
+    {
+      "id": "dataStreamPublisherConfig",
+      "type": "pickString",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "sparkConfigQuery",
+      "type": "promptString",
+      "description": "Config Type(s)"
+    },
+    {
+      "id": "sparkdConfigQuery",
+      "type": "promptString",
+      "description": "Config Type(s)"
+    }
+  ]
+}

CMakeLists.txt

@@ -14,12 +14,17 @@ set(
   CACHE
   STRING
   "External dependencies root folder (should contain folders: lib, bin, include")
+
+# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
+# can run simultaneously without fighting over the same shm files (e.g., sparkd,
+# api server, etc.).
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
 set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
 set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
-set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
+set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
 if (WIN32)
   set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
   set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -31,13 +36,13 @@ endif()
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})
 
-if(BUILD_API_SERVER)
-  add_compile_definitions(SPARK_ENABLE_API_SERVER)
+add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
-
+add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
-  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
-  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
   # subdirs
   add_subdirectory(src/lib)
 else()
@@ -47,13 +52,7 @@ else()
   set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
   set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
 
-  if(BUILD_API_SERVER)
-    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
-    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
-  elseif(BUILD_SPARKD)
-    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
-    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
-
+  if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
     add_compile_definitions(SPARK_ENABLE_CCAPI)

CMakePresets.json

@@ -15,11 +15,13 @@
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build",
+      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -29,8 +31,13 @@
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
       }
     },
 
@@ -46,6 +53,8 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "ON"
@@ -62,7 +71,8 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
       }
     },
 
@@ -95,98 +105,72 @@
     {
       "name": "sparkd-debug-clang",
       "inherits": "sparkd-debug",
-      "displayName": "Debug (clang)",
+      "displayName": "Sparkd (Debug | Clang)",
       "description": "Debug build compiled w/ clang",
-      "binaryDir": "${sourceDir}/build/clang/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
       }
     },
 
     {
       "name": "sparkd-debug-gcc",
       "inherits": "sparkd-debug",
-      "displayName": "Debug (gcc)",
+      "displayName": "Sparkd (Debug | Gcc)",
       "description": "Debug build compiled w/ gcc",
-      "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
       }
     },
 
     {
       "name": "sparkd-release-clang",
       "inherits": "sparkd-release",
-      "displayName": "Release (clang)",
+      "displayName": "Sparkd (Release | Clang)",
       "description": "Release build compiled w/ clang",
-      "binaryDir": "${sourceDir}/build/clang/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
       }
     },
 
     {
       "name": "sparkd-release-gcc",
       "inherits": "sparkd-release",
-      "displayName": "Release (gcc)",
+      "displayName": "Sparkd (Release | Gcc)",
       "description": "Release build compiled w/ gcc",
-      "binaryDir": "${sourceDir}/build/gcc/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
-      }
-    },
-
-    {
-      "name": "spark-api-server-base",
-      "hidden": true,
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
       }
     },
 
     {
       "name": "spark-api-server-windows",
       "inherits": "spark",
-      "displayName": "Spark Api Server",
+      "displayName": "Spark Api Server (Windows)",
       "cacheVariables": {
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        }
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
       }
     },
 
     {
       "name": "spark-api-server-linux",
       "inherits": "sparkd",
-      "displayName": "Spark Api Server",
+      "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
         },
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        },
         "BUILD_SPARKD": {
           "type": "BOOL",
           "value": "OFF"
@@ -197,7 +181,11 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
       }
     },
 
@@ -220,13 +208,62 @@
     },
 
     {
-      "name": "data-stream-suite",
+      "name": "data-stream-suite-windows",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
       "inherits": "spark",
-      "displayName": "Data Stream Suite",
+      "displayName": "Data Stream Suite (Windows)",
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
+    },
+
+    {
+      "name": "data-stream-suite-linux",
+      "displayName": "Data Stream Suite (Linux)",
+      "description": "Data Stream utils ported over from Edge",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "inherits": "sparkd",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
       }
     }
   ],
@@ -244,7 +281,7 @@
 
     {
       "name": "spark-debug",
-      "displayName": "Debug",
+      "displayName": "Spark (Debug)",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Debug"
@@ -252,7 +289,7 @@
 
     {
       "name": "spark-release",
-      "displayName": "Release",
+      "displayName": "Spark (Release)",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Release"
@@ -280,28 +317,28 @@
     {
       "name": "sparkd-debug-clang",
       "configurePreset": "sparkd-debug-clang",
-      "displayName": "Debug (clang)",
+      "displayName": "Sparkd (Debug | Clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-debug-gcc",
       "configurePreset": "sparkd-debug-gcc",
-      "displayName": "Debug (gcc)",
+      "displayName": "Sparkd (Debug | Gcc)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-clang",
       "configurePreset": "sparkd-release-clang",
-      "displayName": "Release (clang)",
+      "displayName": "Sparkd (Release | Clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-gcc",
       "configurePreset": "sparkd-release-gcc",
-      "displayName": "Release (gcc)",
+      "displayName": "Sparkd (Release | Gcc)",
       "inherits": "sparkd"
     },
 
@@ -318,14 +355,14 @@
 
     {
       "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Windows | Debug)",
       "inherits": "spark-api-server-windows",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Windows | Release)",
       "inherits": "spark-api-server-windows",
       "configuration": "Release"
     },
@@ -344,14 +381,14 @@
     {
       "name": "spark-api-server-linux-debug",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-linux-release",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Release)",
+      "displayName": "Spark Api Server (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -371,16 +408,30 @@
     },
 
     {
-      "name": "data-stream-suite-debug",
-      "displayName": "DataStreamSuite (Debug)",
-      "configurePreset": "data-stream-suite",
+      "name": "data-stream-suite-windows-debug",
+      "displayName": "Data Stream Suite (Windows | Debug)",
+      "configurePreset": "data-stream-suite-windows",
       "configuration": "Debug"
     },
 
     {
-      "name": "data-stream-suite-release",
-      "displayName": "DataStreamSuite (Release)",
-      "configurePreset": "data-stream-suite",
+      "name": "data-stream-suite-windows-release",
+      "displayName": "Data Stream Suite (Windows | Release)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "data-stream-suite-linux-debug",
+      "displayName": "Data Stream Suite (Linux | Debug)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-linux-release",
+      "displayName": "Data Stream Suite (Linux | Release)",
+      "configurePreset": "data-stream-suite-linux",
       "configuration": "Release"
     }
   ]

docker/sparkd/dev/Dockerfile

@@ -1,74 +1,65 @@
-FROM ubuntu:22.04
+FROM ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/New_York
 
-# Note installations that take a while (more than 10s)
+# light deps (should be quick to download/install)
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    curl \
+    gnupg \
+    liblttng-ust-dev \
+    libpcap0.8-dev \
+    locales \
+    ninja-build \
+    software-properties-common \
+    sqlite3 \
+    tzdata
 
-RUN apt-get update
-RUN apt-get install -y ca-certificates
-RUN apt-get install -y ccache
-RUN apt-get install -y curl
-RUN apt-get install -y fish
-RUN apt-get install -y gdb
-# git-all: ~250s
-RUN apt-get install -y git-all
-RUN apt-get install -y gnupg
-# liblttng-ust-dev: ~30s
-RUN apt-get install -y liblttng-ust-dev
-RUN apt-get install -y libssh2-1-dev
-RUN apt-get install -y locales
-RUN apt-get install -y ninja-build
-# openssh-server: ~40s
-RUN apt-get install -y openssh-server
-RUN apt-get install -y pkg-config
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y sudo
-RUN apt-get install -y tzdata
-RUN apt-get install -y vim
-RUN apt-get install -y zip
+# clang
+RUN apt-get update && apt-get install -y \
+    clang-13 \
+    clang-format-13 \
+    clangd-13 \
+    clang-15 \
+    clang-format-15 \
+    clangd-15
 
-# Python: ~30s
-RUN apt-get install -y \
+# g++
+RUN apt-get update && apt-get install -y \
+    g++-11 \
+    g++-12
+
+# python
+RUN apt-get update && apt-get install -y \
     python3-pip \
     python3-venv \
     && python3 -m pip install --upgrade pip
 
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-    && apt-get update \
-    && apt-get install -y g++-$gcc_major
+# dotnet
+RUN apt-get update && apt-get install -y \
+    dotnet7
 
-# Clang
-ARG clang_major=17
-RUN curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-    | gpg --dearmor - \
-    | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
-    | tee -a "/etc/apt/sources.list.d/llvm.list" \
-    && apt-get update \
-    && apt-get install -y \
-    clang-$clang_major \
-    clang-format-$clang_major \
-    clangd-$clang_major
+# ssh
+RUN apt-get update && apt-get install -y \
+    libssh2-1-dev \
+    openssh-server
 
-# CMake
-RUN curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+# cmake
+RUN \
+    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
     | gpg --dearmor - \
     | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
     && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
     | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
     && apt-get update \
+    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
+    && apt-get install -y kitware-archive-keyring \
+    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
+    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
-# Dotnet
-RUN curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" \
-    && dpkg -i packages-microsoft-prod.deb \
-    && rm packages-microsoft-prod.deb \
-    && apt-get update \
-    && apt-get install -y dotnet-sdk-8.0
-
-# Node
+# node
 ARG node_major=18
 RUN mkdir -p /etc/apt/keyrings \
     && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
@@ -78,35 +69,60 @@ RUN mkdir -p /etc/apt/keyrings \
     && apt-get update \
     && apt-get install -y nodejs
 
-# Emumba deps
+# ccache is done here so that ccache sees clang++ installed, not clang-10
+RUN apt-get update && apt-get install -y \
+    ccache 
+
+# emumba (light deps)
 RUN apt-get update && apt-get install -y \
     build-essential \
     doxygen \
+    python3-sphinx \
     graphviz \
     libgtest-dev \
     libssl-dev \
-    pkg-config \
-    python3-sphinx \
     rapidjson-dev \
-    zlib1g-dev
+    zlib1g-dev \
+    pkg-config
 
-# Set up default executables (update-alternatives)
-RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+# emumba (boost)
+RUN apt-get update && apt-get install -y \
+    libboost-filesystem-dev \
+    libboost-program-options-dev \
+    libboost-test-dev \
+    libboost-date-time-dev
+
+# dev utils
+RUN apt-get update && apt-get install -y \ 
+    fish \
+    git-all \
+    sudo \
+    vim \
+    wget
+
+# set up default executables (update-alternatives)
+RUN \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
 
 # Add compiler symlinks for ccache 
-RUN ln -s /usr/bin/ccache /usr/local/bin/clang; \
+RUN \
+    ln -s /usr/bin/ccache /usr/local/bin/clang; \
     ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc; \
     ln -s /usr/bin/ccache /usr/local/bin/g++; \
-    ln -s /usr/bin/ccache /usr/local/bin/gcc-$gcc_major; \
-    ln -s /usr/bin/ccache /usr/local/bin/g++-$gcc_major
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-12
 
 RUN locale-gen en_US.UTF-8
 
@@ -114,7 +130,8 @@ ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
 
-RUN groupadd -g "$docker_uid" "$docker_user" \
+RUN \
+    groupadd -g "$docker_uid" "$docker_user" \
     && useradd \
     -m \
     -u "$docker_uid" \

global.json

@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "6.0.200",
+    "rollForward": "latestMajor"
+  }
+}

playground-1.mongodb.js

@@ -1,45 +0,0 @@
-/* global use, db */
-// MongoDB Playground
-// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
-// Make sure you are connected to enable completions and to be able to run a playground.
-// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
-// The result of the last command run in a playground is shown on the results panel.
-// By default the first 20 documents will be returned with a cursor.
-// Use 'console.log()' to print to the debug output.
-// For more documentation on playgrounds please refer to
-// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
-
-// Select the database to use.
-use('mongodbVSCodePlaygroundDB');
-
-// Insert a few documents into the sales collection.
-db.getCollection('sales').insertMany([
-    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
-    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
-]);
-
-// Run a find command to view items sold on April 4th, 2014.
-const salesOnApril4th = db.getCollection('sales').find({
-    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
-}).count();
-
-// Print a message to the output window.
-console.log(`${salesOnApril4th} sales occurred in 2014.`);
-
-// Here we run an aggregation and open a cursor to the results.
-// Use '.toArray()' to exhaust the cursor to return the whole result set.
-// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
-db.getCollection('sales').aggregate([
-    // Find all of the sales that occurred in 2014.
-    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
-    // Group the total sales for each product.
-    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
-]);
-
-

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/Crono.cs

@@ -1,41 +1,22 @@
-using mbts.spark;
+using System.Diagnostics;
 
-using System;
-using System.Runtime;
+namespace BCG.Utilities.General;
 
-namespace BCG.Utilities.General
-{
-	/// <summary>
-	/// Summary description for Crono.
-	/// </summary>
-	public class Crono
-	{
-      public Crono()
-      {
-         myTimer = new HighPerformanceTimer();
-      }
+/// <summary>
+/// Summary description for Crono.
+/// </summary>
+public class Crono {
+   readonly Stopwatch myTimer = new();
 
-      public void start() 
-      {myTimer.start();}
+   public void start() =>
+      myTimer.Start();
 
-      public void stop()
-      {myTimer.stop();}
+   public void stop() =>
+      myTimer.Stop();
 
-      public TimeSpan time
-      {
-         get 
-         {
-            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
-         }
-      }
-
-      public double miliseconds
-      {
-         get
-         {return myTimer.totalMilliseconds;}
-      }
-
-      private HighPerformanceTimer myTimer;
-	}
+   public TimeSpan time =>
+      myTimer.Elapsed;
 
+   public double miliseconds =>
+      myTimer.Elapsed.TotalMilliseconds;
 }

src/DataStreamSuite/BlueRegistry/BlueRegistry.cs

@@ -1,192 +1,192 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Reflection;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public enum RegistryRoot { HKCU, HKLM }
-
-   [Serializable]
-   public class BlueRegistry
-   {
-      public const string BlueSubKey = @"Software\Blue Capital Group\";
-      public static readonly string ApplicationPrefix;
-      protected string myFullPath;
-
-      public BlueRegistry(string subPath)
-      {
-         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
-      }
-
-      public BlueRegistry(BlueRegistry parent, string subPath)
-      {
-         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
-      }
-
-      static BlueRegistry()
-      {
-         string app = "Applications";
-
-         Assembly ass = Assembly.GetEntryAssembly();
-         if (ass != null)
-         {
-            app = ass.GetName().Name;
-
-            object[] attrs = ass.GetCustomAttributes(
-               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
-
-            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
-            {
-               app = ((AssemblyTitleAttribute)attrs[0]).Title;
-            }
-         }
-
-         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
-      }
-
-      public object getValue(string name)
-      {
-         return getValue(RegistryRoot.HKCU, name);
-      }
-
-      public object getValue(string name, object defaultValue)
-      {
-         return getValue(RegistryRoot.HKCU, name, defaultValue);
-      }
-
-      public void setValue(string name, object value)
-      {
-         setValue(RegistryRoot.HKCU, name, value);
-      }
-
-      public void deleteValue(string name)
-      {
-         deleteValue(RegistryRoot.HKCU, name);
-      }
-
-      private RegistryKey myGetRegistryKey(RegistryRoot root)
-      {
-         if (root == RegistryRoot.HKCU)
-            return Registry.CurrentUser.CreateSubKey(myFullPath);
-         else
-            return Registry.LocalMachine.CreateSubKey(myFullPath);
-      }
-
-      #region Generic Methods
-      internal virtual void setValue(RegistryRoot root, string name, object value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.SetValue(name, value);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name);
-         }
-      }
-
-      internal virtual void deleteValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.DeleteValue(name);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name, defaultValue);
-         }
-      }
-      #endregion
-
-      #region Double
-      internal virtual void setDouble(RegistryRoot root, string name, double value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            DoubleRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual double getDouble(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return DoubleRegistryHandler.getValue(rk, name);
-         }
-      }
-      #endregion
-
-      #region Enum
-      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            EnumRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return EnumRegistryHandler.getValue(rk, name, enumType);
-         }
-      }
-      #endregion
-
-      #region Array
-      internal virtual void setArray(RegistryRoot root, string name, Array array)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            ArrayRegistryHandler.setValue(rk, name, array);
-         }
-      }
-
-      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return ArrayRegistryHandler.getValue(rk, name, elementType);
-         }
-      }
-      #endregion
-
-      #region SortedList
-      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.setValue(rk, name, slist);
-         }
-      }
-
-      internal virtual SortedList getSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return SortedListRegistryHandler.getValue(rk, name);
-         }
-      }
-
-      internal virtual void deleteSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.deleteValue(rk, name);
-         }
-      }
-      #endregion
-   }
-}
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      public virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      public virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,10 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-    <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>
 
 </Project>

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -42,6 +42,7 @@ OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
 Negate                   '!'
+MapsTo                   ':'
 IsOfType                 '/'
 
 %%
@@ -97,6 +98,10 @@ IsOfType                 '/'
    return (int)Token.NEGATE;
 }
 
+{MapsTo} {
+   return (int)Token.MAPS_TO;
+}
+
 {Equal} {
    return (int)Token.EQUAL;
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj

@@ -1,15 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
-    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.DataStream.Websocket/ClientSubscription.cs

@@ -1,9 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-class ClientSubscription(WebsocketClientID id, SubscriptionParams @params, IDisposable dispose) : IDisposable {
-   public SubscriptionParams Params => @params;
-   public WebsocketClientID Id => id;
-
-   public void Dispose() =>
-      dispose.Dispose();
-}

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -1,301 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.collections;
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamExpressions;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-using Spark.Core.Json;
-
-using System.Buffers;
-using System.Reactive.Subjects;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-/// <summary>
-/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
-/// requests (from clients) as input, and emits data/notifications (to clients) as output.
-/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
-/// </summary>
-public class DataSync(JsonSerializerOptions? serializerOpts = null) :
-   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
-   protected readonly object lockObj = new();
-   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
-   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
-
-   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
-
-   public void OnCompleted() =>
-      Log.Api.info("DataSync incoming request stream has completed.");
-
-   public void OnError(Exception error) =>
-      Log.Api.error($"DataSync error: {error}");
-
-   public void OnNext(WebsocketConnectionUpdate value) {
-      switch (value.Type) {
-         case WebsocketConnectionUpdateType.IncomingMessage:
-            onIncomingMessage(value.IncomingMessage);
-            break;
-         case WebsocketConnectionUpdateType.Connect:
-            onConnect(value.Connect);
-            break;
-         case WebsocketConnectionUpdateType.Disconnect:
-            onDisconnect(value.Disconnect);
-            break;
-         default:
-            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
-            break;
-      }
-   }
-
-   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
-      lock (lockObj) {
-         requestSend += observer.OnNext;
-         return Disposable.Create(Dispose);
-      }
-      void Dispose() {
-         lock (lockObj) {
-            requestSend -= observer.OnNext;
-         }
-      }
-   }
-
-   static JsonRpcId? extractRpcId(string messageBody) {
-      try {
-         return JsonNode.Parse(messageBody)
-            ?.AsObject()
-            ?.GetValueOrDefault("id")
-            ?.Deserialize<JsonRpcId>();
-      } catch {
-         return null;
-      }
-   }
-
-   static string getRpcNotification(DataPacket dp) {
-      using (var buffer = new MemoryStream())
-      using (var writer = new Utf8JsonWriter(buffer))
-      using (writer.WriteObject()) {
-         writer.WriteString("method", "subscription");
-         using (writer.WriteObject("params")) {
-            writer.Write("tableName", dp.schema.tableName);
-            writer.Write("columns", dp.schema.columnInfo);
-            var cellJsonFactories = dp.schema.columnInfo.Select(getCellJsonFactory).ToList();
-            using (writer.WriteArray("added")) {
-               foreach (IBlueRow row in (dp.addedRows?.Cast<IBlueRow>()).empty_if_null()) {
-                  using (writer.WriteObject()) {
-                     for (int i = 0; i < cellJsonFactories.Count; i++) {
-                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
-                     }
-                  }
-               }
-            }
-            using (writer.WriteArray("deleted")) {
-               foreach (IBlueRow row in (dp.deletedRows?.Cast<IBlueRow>()).empty_if_null()) {
-                  using (writer.WriteObject()) {
-                     for (int i = 0; i < cellJsonFactories.Count; i++) {
-                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
-                     }
-                  }
-               }
-            }
-         }
-         writer.WriteString("jsonrpc", "2.0");
-         writer.Flush();
-         return Encoding.UTF8.GetString(buffer.ToArray());
-      }
-   }
-
-   static Func<IBlueRow, JsonNode?> getCellJsonFactory(ColumnInfo colInfo) {
-      var t = colInfo.colType;
-      if (t == typeof(int)) {
-         return NullCheckWrapper(v => (int)v);
-      }
-      if (t == typeof(uint)) {
-         return NullCheckWrapper(v => (uint)v);
-      }
-      if (t == typeof(long)) {
-         return NullCheckWrapper(v => (long)v);
-      }
-      if (t == typeof(ulong)) {
-         return NullCheckWrapper(v => (ulong)v);
-      }
-      if (t == typeof(short)) {
-         return NullCheckWrapper(v => (short)v);
-      }
-      if (t == typeof(ushort)) {
-         return NullCheckWrapper(v => (ushort)v);
-      }
-      if (t == typeof(double)) {
-         return NullCheckWrapper(v => (double)v);
-      }
-      if (t == typeof(float)) {
-         return NullCheckWrapper(v => (float)v);
-      }
-      if (t == typeof(bool)) {
-         return NullCheckWrapper(v => (bool)v);
-      }
-      return NullCheckWrapper(v => (string)v);
-
-      Func<IBlueRow, JsonNode?> NullCheckWrapper(Func<object, JsonNode> toJson) =>
-         r => {
-            var obj = r[colInfo.colNo];
-            if (obj is null) {
-               return null;
-            }
-            return toJson(obj);
-         };
-   }
-
-   void onConnect(WebsocketConnectNotification notification) {
-      var id = notification.Id;
-      lock (lockObj) {
-         clients.Add(id, []);
-      }
-      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
-      if (notification.Message is not null) {
-         onIncomingMessage(notification.Message);
-      }
-   }
-
-   void onDisconnect(WebsocketDisconnectNotification notification) {
-      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
-      unsubscribeAll(notification.Id);
-      clients.Remove(notification.Id);
-   }
-
-   void onIncomingMessage(WebsocketIncomingMessage message) {
-      var id = message.sender_client_id;
-      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
-         onBadRequest(message, err);
-         return;
-      }
-      switch (req.Method!.ToLower()) {
-         case "subscribe":
-            subscribe(id, req);
-            break;
-         case "publish":
-            Log.Api.error("DataStreamWebsocket api 'publish' not implemented.");
-            // TODO
-            break;
-         case "unsubscribe":
-            unsubscribe(id, req);
-            break;
-         case "unsubscribe_all":
-            unsubscribeAll(id);
-            break;
-         default:
-            Log.Api.error($"Received request of unknown type: {req}");
-            break;
-      }
-   }
-
-   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
-      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send([new(id, rpcResStr)]);
-   }
-
-   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
-      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send([new(id, rpcResStr)]);
-   }
-
-   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
-      lock (lockObj) {
-         if (requestSend is null) {
-            Log.Api.warn(
-               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
-            return false;
-         }
-         requestSend(messages);
-         return true;
-      }
-   }
-
-   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
-      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
-      lock (lockObj) {
-         clients[id].Add(sub);
-         sendResponse(id, req.Id!.Value, req.Params);
-         stream.onChange += OnChange;
-      }
-
-      void OnChange(object? newValue) {
-         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
-         var notif = getRpcNotification(dp);
-         send([new(id, notif)]);
-      }
-
-      void Dispose() =>
-         stream.onChange -= OnChange;
-   }
-
-   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>();
-      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
-      // Minimize amount of code inside lock (just need it to find/remove the subscription)
-      List<ClientSubscription>? subs = null;
-      ClientSubscription? sub = null;
-      lock (lockObj) {
-         subs = clients.GetValueOrDefault(id);
-         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
-      }
-      if (subs is null) {
-         // Didn't find client (log/throw).
-         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      if (sub is null) {
-         // Found client, but didn't find subscription (log/throw).
-         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
-            $"Id: {id}. Request:\n{subParams}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      // Found client and subscription (dispose/remove subscription).
-      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
-      sub.Dispose();
-   }
-
-   void unsubscribeAll(WebsocketClientID id) {
-      lock (lockObj) {
-         if (!clients.TryGetValue(id, out var subs)) {
-            throw new Exception(
-               $"Unable to unsubscribe: did not find client with id {id}");
-         }
-         subs.DisposeAll();
-         subs.Clear();
-      }
-   }
-
-   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
-      var rpcId = extractRpcId(req.message_body);
-      var logMsg = $"Recieved invalid request:\n{req.message_body}";
-      if (errorMsg is not null) {
-         logMsg += $"\n\n**Exception**\n{errorMsg}";
-      }
-      Log.Api.error(logMsg);
-      if (rpcId is null) {
-         return;
-      }
-      // Send error back to client
-      var rpcErr = JsonRpcError.Create(
-         (long)JsonRpcErrorCode.InvalidRequest,
-         $"Received invalid request: {errorMsg ?? "unknown error"}",
-         req.message_body);
-      sendError(req.sender_client_id, rpcId.Value, rpcErr);
-   }
-}

src/DataStreamSuite/Bts.DataStream.Websocket/JObjectExtensions.cs

@@ -1,34 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-using Newtonsoft.Json.Linq;
-
-using System.Diagnostics.CodeAnalysis;
-
-public static class JObjectExtensions {
-   public static bool TryParse<T>(
-      this JObject filter, string key, [MaybeNull] out T obj, [NotNullWhen(false)] out string? error) {
-      obj = default;
-      var value = filter.GetValueOrDefault(key)?.ToString();
-      if (value == null) {
-         error = $"{key} not present";
-         return true;
-      }
-      return WebsocketJsonParsers.TryParse(value, key, out obj, out error);
-   }
-
-   public static bool TryParse<T>(
-     this JObject filter, string key, out T result, [NotNullWhen(false)] out string? error, T defaultValue) {
-      var value = filter.GetValueOrDefault(key)?.ToString();
-      if (value == null) {
-         result = defaultValue;
-         error = $"{key} not present";
-         return true;
-      }
-      if (WebsocketJsonParsers.TryParse<T>(value, key, out var maybeRes, out error)) {
-         result = maybeRes ?? defaultValue;
-         return true;
-      }
-      result = defaultValue;
-      return false;
-   }
-}

src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs

@@ -1,48 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class MultilineStringJsonConverter : JsonConverter<string> {
-   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (typeToConvert != typeof(string)) {
-         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-      }
-      return reader.TokenType switch {
-         JsonTokenType.String => StringImpl(ref reader),
-         JsonTokenType.StartArray => ArrayImpl(ref reader),
-         _ => throw new ArgumentException(
-            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-      };
-
-      static string? StringImpl(ref Utf8JsonReader reader) =>
-         new(reader.GetString()!);
-
-      static string? ArrayImpl(ref Utf8JsonReader reader) {
-         var lines = new List<string>(2);
-         while (reader.Read()) {
-            switch (reader.TokenType) {
-               case JsonTokenType.String:
-                  lines.Add(reader.GetString()!);
-                  break;
-               case JsonTokenType.EndArray:
-                  return string.Join("\n", lines);
-            }
-         }
-         throw new JsonException(
-            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-      }
-   }
-
-   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
-      var lines = value.Split("\n");
-      if (lines.Length < 2) {
-         writer.WriteRawValue(value);
-         return;
-      }
-      writer.WriteRawValue($"[{", ".join(lines)}]");
-   }
-}

src/DataStreamSuite/Bts.DataStream.Websocket/RequestType.cs

@@ -1,8 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-public enum RequestType {
-   Unspecified,
-   Subscribe,
-   Unsubscribe,
-   UnsubscribeAll,
-}

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -1,25 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-using bts;
-
-using Google.Protobuf;
-
-using System.Text.Json;
-
-public static class Session {
-   static int sessionIdCounter;
-
-   public static async Task Start(
-      WebsocketServerConfig config, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      var sessionId = Interlocked.Increment(ref sessionIdCounter);
-      var gateway = new WebsocketClient(config);
-      var sync = new DataSync(opts);
-      using (gateway.Subscribe(sync))
-      using (sync.Subscribe(gateway)) {
-         Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
-         Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
-         await gateway.ConnectAsync();
-      }
-   }
-}

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -1,112 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-using bts.utils;
-
-using Spark.Core;
-using Spark.Core.Json;
-
-using System;
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionParams> {
-   static class PropNames {
-      public const string QUERY = "query";
-   }
-
-   // Serializer supports plain string, or an array of lines for convenience
-   [MaybeNull]
-   [JsonPropertyName(PropNames.QUERY)]
-   [JsonConverter(typeof(MultilineStringJsonConverter))]
-   public string? Query { get; set; } = query;
-
-   public bool Equals(SubscriptionParams? other) =>
-      other is not null &&
-      // Not the most elegant equals impl, but it'll work
-      Query.safe_equals(other.Query, (a, b) => a?.ToString() == b?.ToString());
-
-   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
-      public override bool CanConvert(Type typeToConvert) =>
-         typeToConvert == typeof(SubscriptionParams);
-
-      public override SubscriptionParams Read(
-         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-         if (typeToConvert != typeof(SubscriptionParams)) {
-            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-         }
-         return read(ref reader);
-      }
-
-      static SubscriptionParams read(ref Utf8JsonReader reader) {
-         return reader.TokenType switch {
-            JsonTokenType.String => StringImpl(ref reader),
-            JsonTokenType.StartArray => ArrayImpl(ref reader),
-            JsonTokenType.StartObject => ObjectImpl(ref reader),
-            _ => throw new ArgumentException(
-               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-         };
-
-         static SubscriptionParams StringImpl(ref Utf8JsonReader reader) =>
-            new(reader.GetString()!);
-
-         static SubscriptionParams ArrayImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-
-         static SubscriptionParams ObjectImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.PropertyName:
-                     var propName = reader.GetString()!;
-                     if (propName == PropNames.QUERY) {
-                        switch (reader.TokenType) {
-                           case JsonTokenType.StartArray:
-                              return ArrayImpl(ref reader);
-                           case JsonTokenType.String:
-                              return StringImpl(ref reader);
-                        }
-                     }
-                     break;
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-      }
-
-      public override void Write(
-         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
-         var q = value.Query;
-         var lines = q?.Split("\n");
-         if (lines is null || lines.Length < 2) {
-            writer.WriteString(PropNames.QUERY, value.Query);
-            return;
-         }
-         writer.WriteArray(PropNames.QUERY, lines);
-      }
-   }
-
-   public override bool Equals(object? obj) =>
-      Equals(obj as SubscriptionParams);
-
-   public override int GetHashCode() =>
-      Query?.Length ?? typeof(SubscriptionParams).GetHashCode();
-}

src/DataStreamSuite/Bts.DataStream.Websocket/Usings.cs

@@ -1,5 +0,0 @@
-global using bts.websocket_core;
-
-global using System;
-
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs

@@ -1,41 +0,0 @@
-#nullable enable
-using BCG;
-
-using Spark.Core.Json;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Bts.DataStreamCore;
-
-class ViewStreamUpdate {
-   [JsonPropertyName("tableName")]
-   public string? TableName { get; set; }
-
-   [JsonPropertyName("columns")]
-   public List<ColumnInfo>? Columns { get; set; }
-
-   [JsonPropertyName("addedRows")]
-   public List<Dictionary<string, object?>>? AddedRows { get; set; }
-
-   [JsonPropertyName("deletedRows")]
-   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
-
-   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
-      var schema = new BlueTableSchema([.. Columns!], TableName);
-      var addedRowsTable = new BlueTable(schema);
-      var deletedRowsTable = new BlueTable(schema);
-      if (applyChanges) {
-         state.deleteRows(deletedRowsTable);
-         state.addRows(addedRowsTable);
-      }
-      return new(state, addedRowsTable, deletedRowsTable);
-   }
-
-   public static string ToJson() {
-      return "";
-   }
-}

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -1,18 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Bts.DataStream.Websocket\Bts.DataStream.Websocket.csproj" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServer.conf

@@ -1,7 +0,0 @@
-{
-  "host": "localhost",
-  "port": 8080,
-  "scheme": "ws",
-  "default_end_point": "/",
-  "keep_alive_interval_secs": 1
-}

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs

@@ -1,76 +0,0 @@
-using bts.api.core;
-
-using Bts.DataStream.Websocket;
-
-using Google.Protobuf;
-
-using System.CommandLine.Parsing;
-using System.Text.Json;
-
-static WebsocketServerConfig readConfigFile(string path) {
-   using var fs = File.OpenRead(path);
-   using var sr = new StreamReader(fs);
-   return JsonParser.Default.Parse<WebsocketServerConfig>(sr);
-}
-
-static WebsocketServerConfig? readConfigArg(ArgumentResult result) {
-   var path = result.Tokens.Select(t => t.Value).FirstOrDefault() ?? string.Empty;
-   return readConfigFile(path);
-}
-
-static WebsocketServerConfig getDefaultConfig() {
-   const string defaultCfgPath = "DataStreamWebsocketServer.conf";
-   if (File.Exists(defaultCfgPath)) {
-      return readConfigFile(defaultCfgPath);
-   }
-   return new() {
-      Scheme = "http",
-      Host = "localhost",
-      Port = 8080,
-      DefaultEndPoint = "/",
-      KeepAliveIntervalSecs = 1
-   };
-}
-   
-static async Task run(
-   WebsocketServerConfig? wssConfig, string user, string password, string? sparkdConfigSource) {
-   var opts = new JsonSerializerOptions(JsonSerializerDefaults.Web);
-   wssConfig ??= getDefaultConfig();
-   Console.WriteLine("Starting DataStreamWebsocketServer...");
-   var serviceCtx = ServiceContext.create(sparkdConfigSource);
-   await serviceCtx.login(user, password);
-   await Session.Start(wssConfig, opts);
-   // if (sparkdConfigSource is not null && sparkdConfigSource.Length > 9999) {
-   // } else {
-   //    await ChatGptSession.Start();
-   // }
-}
-
-var userOpt = new Option<string>("--user", () => "bts");
-var passOpt = new Option<string>("--password", () => string.Empty);
-var sparkdConfigSourceOpt = new Option<string>(
-   "--sparkd", description: "The sparkd config server url");
-// var bling = new ParseArgument<WebsocketServerConfig?>(parseWsConfig);
-
-// WebsocketServerConfig? parseWsConfig(ArgumentResult result) {
-//    Token t;
-//    TokenType tt;
-//    Symbol s;
-//    throw new NotImplementedException();
-// }
-
-var configOpt = new Option<WebsocketServerConfig?>(
-   "--config", readConfigArg, description: "WebsocketServerConfig file path");
-configOpt.SetDefaultValue(new());
-var rootCommand = new RootCommand("Starts a DataStream Websocket Server");
-rootCommand.AddOption(configOpt);
-rootCommand.AddOption(userOpt);
-rootCommand.AddOption(passOpt);
-rootCommand.SetHandler(run, configOpt, userOpt, passOpt, sparkdConfigSourceOpt);
-
-try {
-   return await rootCommand.InvokeAsync(args);
-} catch (Exception ex) {
-   Console.WriteLine($"Error: {ex}");
-}
-return 1;

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs

@@ -1,6 +0,0 @@
-global using bts.websocket_core;
-global using bts.websocket_core.json_rpc;
-
-global using System.CommandLine;
-
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -1,20 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.2' } },
-  plugins: ['react-refresh'],
-  rules: {
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -1,11 +0,0 @@
-<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
-  <PropertyGroup>
-    <StartupCommand>npm run dev</StartupCommand>
-    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
-    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
-    <!-- Allows the build (or compile) script located on package.json to run on Build -->
-    <ShouldRunBuildScript>false</ShouldRunBuildScript>
-    <!-- Folder where production build objects will be placed -->
-    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\dist</PublishAssetsDirectory>
-  </PropertyGroup>
-</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -1,8 +0,0 @@
-# React + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
-- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
-- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -1,13 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite + React</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.jsx"></script>
-  </body>
-</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-  </packageSources>
-  <disabledPackageSources>
-    <clear />
-  </disabledPackageSources>
-</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -1,27 +0,0 @@
-{
-  "name": "bts.datastreamwebapp.client",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
-    "preview": "vite preview"
-  },
-  "dependencies": {
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "react-use-websocket": "^4.5.0"
-  },
-  "devDependencies": {
-    "@types/react": "^18.2.43",
-    "@types/react-dom": "^18.2.17",
-    "@vitejs/plugin-react": "^4.2.1",
-    "eslint": "^8.55.0",
-    "eslint-plugin-react": "^7.33.2",
-    "eslint-plugin-react-hooks": "^4.6.0",
-    "eslint-plugin-react-refresh": "^0.4.5",
-    "vite": "^5.0.8"
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -1,44 +0,0 @@
-#root {
-    max-width: 1280px;
-    margin: 0 auto;
-    padding: 2rem;
-    text-align: center;
-    --cell-background-color: #293136;
-    --cell-color: #cbe3ef;
-}
-
-tr:nth-child(even) {
-    background: #F2F2F2;
-}
-
-tr:nth-child(odd) {
-    background: #FFF;
-}
-
-th, td {
-    padding-left: 1rem;
-    padding-right: 1rem;
-    background-color: var(--cell-background-color);
-    color: var(--cell-color);
-}
-
-textarea#query-input {
-  height: 5em;
-  width: 50em;
-}
-
-button#query-button {
-  height: 5em;
-  vertical-align: top;
-}
-
-table#query-output {
-  float: none;
-  clear: both;
-}
-
-p#footer {
-  float: none;
-  clear: both;
-  vertical-align: bottom;
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -1,88 +0,0 @@
-import { Component } from "react";
-import "./App.css";
-
-export class Home extends Component {
-  static displayName = Home.name;
-  static sparkdHost = "localhost";
-
-  render() {
-    let socket = null;
-    const wsUrl = `ws://localhost:5268/ws`;
-
-    function connectToSparkd() {
-      if (socket == null) {
-        socket = new WebSocket(wsUrl);
-      }
-      if (socket && socket.readyState === WebSocket.OPEN) {
-        return;
-      }
-
-      const output = document.getElementById("query-output");
-      socket.onopen = (event) => {
-        console.log(`Connected to ${wsUrl}`);
-      };
-
-      socket.onmessage = (event) => {
-        console.log(`Received message: ${event.data}`);
-        output.innerHTML = event.data;
-      };
-
-      socket.onerror = (error) => {
-        console.log(`Error on websocket ${wsUrl}: ${error}`);
-      };
-
-      socket.onclose = (event) => {
-        if (event.wasClean) {
-          const msg = `Connection to ${wsUrl} closed cleanly, code=${event.code}, reason=${event.reason}`;
-          console.log(msg);
-        } else {
-          const msg = `Connection to ${wsUrl} abruptly closed`;
-          console.log(msg);
-        }
-      };
-    }
-
-    function executeQuery() {
-      if (socket == null || !socket) {
-        connectToSparkd();
-      }
-      if (socket == null || !socket) {
-        console.log("Failed to connect to websocket server");
-        return;
-      }
-      if (socket.readyState === WebSocket.OPEN) {
-        const queryInput = document.getElementById("query-input");
-        const rpcReq = {
-          jsonrpc: "2.0",
-          method: "subscribe",
-          params: {
-            query: queryInput.value,
-          },
-          id: 1,
-        };
-        socket.send(JSON.stringify(rpcReq));
-      }
-    }
-
-    return (
-      <div>
-        <h1>Data Stream Query Analyzer</h1>
-        <p>Run queries against the BTS Cloud</p>
-        <div>
-          <textarea id="query-input" placeholder="Enter Query"></textarea>
-          <button id="query-button" onClick={executeQuery}>
-            Execute
-          </button>
-        </div>
-        <table id="query-output"></table>
-        <p id="footer">
-          The <code>ClientApp</code> subdirectory is a standard React
-          application based on the <code>create-react-app</code> template. If
-          you open a command prompt in that directory, you can run{" "}
-          <code>npm</code> commands such as <code>npm test</code> or{" "}
-          <code>npm install</code>.
-        </p>
-      </div>
-    );
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -1,68 +0,0 @@
-:root {
-  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
-  line-height: 1.5;
-  font-weight: 400;
-
-  color-scheme: light dark;
-  color: rgba(255, 255, 255, 0.87);
-  background-color: #242424;
-
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-a {
-  font-weight: 500;
-  color: #646cff;
-  text-decoration: inherit;
-}
-a:hover {
-  color: #535bf2;
-}
-
-body {
-  margin: 0;
-  display: flex;
-  place-items: center;
-  min-width: 320px;
-  min-height: 100vh;
-}
-
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-button {
-  border-radius: 8px;
-  border: 1px solid transparent;
-  padding: 0.6em 1.2em;
-  font-size: 1em;
-  font-weight: 500;
-  font-family: inherit;
-  background-color: #1a1a1a;
-  cursor: pointer;
-  transition: border-color 0.25s;
-}
-button:hover {
-  border-color: #646cff;
-}
-button:focus,
-button:focus-visible {
-  outline: 4px auto -webkit-focus-ring-color;
-}
-
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #ffffff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -1,10 +0,0 @@
-import React from 'react'
-import ReactDOM from 'react-dom/client'
-import {Home} from './App.jsx'
-import './index.css'
-
-ReactDOM.createRoot(document.getElementById('root')).render(
-  <React.StrictMode>
-    <Home />
-  </React.StrictMode>,
-)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -1,60 +0,0 @@
-import { fileURLToPath, URL } from 'node:url';
-
-import { defineConfig } from 'vite';
-import plugin from '@vitejs/plugin-react';
-import fs from 'fs';
-import path from 'path';
-import child_process from 'child_process';
-
-const baseFolder =
-    process.env.APPDATA !== undefined && process.env.APPDATA !== ''
-        ? `${process.env.APPDATA}/ASP.NET/https`
-        : `${process.env.HOME}/.aspnet/https`;
-
-const certificateArg = process.argv.map(arg => arg.match(/--name=(?<value>.+)/i)).filter(Boolean)[0];
-const certificateName = certificateArg ? certificateArg.groups.value : "bts.datastreamwebapp.client";
-
-if (!certificateName) {
-    console.error('Invalid certificate name. Run this script in the context of an npm/yarn script or pass --name=<<app>> explicitly.')
-    process.exit(-1);
-}
-
-const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
-const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
-
-if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
-    if (0 !== child_process.spawnSync('dotnet', [
-        'dev-certs',
-        'https',
-        '--export-path',
-        certFilePath,
-        '--format',
-        'Pem',
-        '--no-password',
-    ], { stdio: 'inherit', }).status) {
-        throw new Error("Could not create certificate.");
-    }
-}
-
-// https://vitejs.dev/config/
-export default defineConfig({
-    plugins: [plugin()],
-    resolve: {
-        alias: {
-            '@': fileURLToPath(new URL('./src', import.meta.url))
-        }
-    },
-    server: {
-        proxy: {
-            '^/weatherforecast': {
-                target: 'https://localhost:7128/',
-                secure: false
-            }
-        },
-        port: 5173,
-        https: {
-            key: fs.readFileSync(keyFilePath),
-            cert: fs.readFileSync(certFilePath),
-        }
-    }
-})

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -1,35 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <InvariantGlobalization>true</InvariantGlobalization>
-    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
-    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
-    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy">
-      <Version>8.*-*</Version>
-    </PackageReference>
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
-  </ItemGroup>
-
-  <!-- <ItemGroup>
-    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
-  </ItemGroup> -->
-</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.http

@@ -1,6 +0,0 @@
-@Bts.DataStreamWebApp.Server_HostAddress = http://localhost:5268
-
-GET {{Bts.DataStreamWebApp.Server_HostAddress}}/weatherforecast/
-Accept: application/json
-
-###

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,107 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.api.core;
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamExpressions;
-
-using Microsoft.AspNetCore.Mvc;
-
-using Spark.Core.Disposables;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-
-[Route("/ws")]
-public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
-   readonly ILogger<DataStreamSubscriptionController> logger = logger;
-   IDisposable? streamSubscription;
-
-   [Route("/ws")]
-   public async Task Get() {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      Console.WriteLine("Establishing websocket connection...");
-      using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-      Console.WriteLine($"Websocket connection established.");
-      await runWebSocketServer(null, "bts", string.Empty, webSocket);
-   }
-
-   public static async Task Echo(WebSocket webSocket) {
-      var buffer = new byte[1024 * 4];
-      var receiveResult = await webSocket.ReceiveAsync(
-          new ArraySegment<byte>(buffer), CancellationToken.None);
-
-      while (!receiveResult.CloseStatus.HasValue) {
-         await webSocket.SendAsync(
-             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-
-         receiveResult = await webSocket.ReceiveAsync(
-             new ArraySegment<byte>(buffer), CancellationToken.None);
-      }
-
-      await webSocket.CloseAsync(
-          receiveResult.CloseStatus.Value,
-          receiveResult.CloseStatusDescription,
-          CancellationToken.None);
-   }
-
-   async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
-      var context = ServiceContext.create(sparkd);
-      await context.login(user, password);
-      var buffer = new byte[1024 * 4];
-      var receiveResult = await webSocket.ReceiveAsync(
-          new ArraySegment<byte>(buffer), CancellationToken.None);
-
-      try {
-         while (!receiveResult.CloseStatus.HasValue) {
-            receiveResult = await webSocket.ReceiveAsync(
-                new ArraySegment<byte>(buffer), CancellationToken.None);
-            var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
-            var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
-            var query = rpcReq?.Params?.Query ?? throw new InvalidOperationException(
-               "Failed to parse JsonRpcRequest.Params");
-            var exp = DataStreamExpressionsInterpreter.Evaluate(query);
-            if (exp is null) {
-               Console.WriteLine($"Invalid query expression: {query}.");
-               continue;
-            }
-            var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
-            Disposable.dispose(ref streamSubscription);
-            vs.onChange += OnStreamChanged;
-            streamSubscription = Disposable.Create(() => vs.onChange -= OnStreamChanged);
-         }
-      } catch (Exception ex) {
-         Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
-         return 1;
-      }
-
-      return 0;
-
-      async void OnStreamChanged(object newValue) {
-         if (newValue is not DataPacket dp) {
-            Console.WriteLine(
-               $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
-            return;
-         }
-         Console.WriteLine("Received stream update.");
-         var added_rows_str = dp.addedRows.get_printed_rows(val => val.Length);
-         var msg = $"Received new rows:\n{added_rows_str}";
-         var encoded = Encoding.UTF8.GetBytes(msg);
-         var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
-         await webSocket.SendAsync(
-             payload,
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-      }
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/WeatherForecastController.cs

@@ -1,22 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-using Microsoft.AspNetCore.Mvc;
-
-[ApiController]
-[Route("[controller]")]
-public class WeatherForecastController(ILogger<WeatherForecastController> logger) : ControllerBase {
-   static readonly string[] Summaries = [
-      "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
-   ];
-
-   readonly ILogger<WeatherForecastController> _logger = logger;
-
-   [HttpGet(Name = "GetWeatherForecast")]
-   public IEnumerable<WeatherForecast> Get() {
-      return Enumerable.Range(1, 5).Select(index => new WeatherForecast {
-         Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)),
-         TemperatureC = Random.Shared.Next(-20, 55),
-         Summary = Summaries[Random.Shared.Next(Summaries.Length)]
-      })
-      .ToArray();
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using System.Text.Json.Serialization;
-
-public class DataStreamSubscription {
-   [JsonPropertyName("query")]
-   public string? Query { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamUpdate.cs

@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using BCG;
-
-class DataStreamUpdate {
-   public List<IBlueRow> addedRows { get; set; } = new(0);
-   public List<IBlueRow> deletedRows { get; set; } = new(0);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerOptions.cs

@@ -1,14 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using CommandLine;
-
-class DataStreamWebAppServerOptions {
-   [Option("sparkd")]
-   public string? sparkd { get; set; }
-
-   [Option("user", Default = "bts")]
-   public string? user { get; set; }
-
-   [Option("password", Default = "")]
-   public string? password { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebsocketHandler.cs

@@ -1,58 +0,0 @@
-using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.Logging;
-
-using System;
-using System.Net.WebSockets;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace Bts.DataStreamWebApp.Server {
-   // public class DataStreamWebSocketHandler {
-   //    readonly WebSocketConnectionManager _connectionManager = new();
-   //    readonly ILogger<DataStreamWebSocketHandler> _logger;
-
-   //    public DataStreamWebSocketHandler(
-   //       WebSocketConnectionManager connectionManager,
-   //       ILogger<DataStreamWebSocketHandler> logger
-   //    ) {
-   //       _connectionManager = connectionManager;
-   //       _logger = logger;
-   //    }
-
-   //    public async Task HandleWebSocket(HttpContext context, WebSocket webSocket) {
-   //       var socketId = _connectionManager.AddSocket(webSocket);
-
-   //       _logger.LogInformation($"WebSocket connection established with ID {socketId}");
-
-   //       while (webSocket.State == WebSocketState.Open) {
-   //          var message = await ReceiveMessageAsync(webSocket);
-   //          if (message != null) {
-   //             _logger.LogInformation($"Received message from ID {socketId}: {message}");
-   //             await BroadcastMessageAsync(message);
-   //          }
-   //       }
-
-   //       _connectionManager.RemoveSocket(socketId);
-   //       _logger.LogInformation($"WebSocket connection closed with ID {socketId}");
-   //    }
-
-   //    async Task<string?> ReceiveMessageAsync(WebSocket webSocket) {
-   //       var buffer = new byte[1024];
-   //       var result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
-
-   //       if (result.CloseStatus.HasValue) {
-   //          return null;
-   //       }
-
-   //       return System.Text.Encoding.UTF8.GetString(buffer, 0, result.Count);
-   //    }
-
-   //    async Task BroadcastMessageAsync(string message) {
-   //       foreach (var socket in _connectionManager.GetAllSockets()) {
-   //          if (socket.Value.State == WebSocketState.Open) {
-   //             await socket.Value.SendAsync(System.Text.Encoding.UTF8.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
-   //          }
-   //       }
-   //    }
-   // }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,159 +0,0 @@
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.utils;
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamExpressions;
-using Bts.DataStreamWebApp.Server;
-using Bts.DataStreamWebApp.Server.Controllers;
-
-using CommandLine;
-
-using mbts.config;
-
-using System.Net.WebSockets;
-using System.Reactive.Disposables;
-using System.Reactive.Linq;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-static string fmtReq(HttpRequest req) =>
-   $"{req.Protocol}://{req.Host}/{req.Path}?{req.QueryString}";
-
-static string fmtDataPacket(DataPacket dp) {
-   var sb = new StringBuilder();
-   var rows = dp.addedRows.Cast<IBlueRow>().ToList();
-   var colWidths = dp.schema.columnInfo.Select(c => rows.Max(r => r[c.colNo]?.ToString()?.Length ?? 0)).ToArray();
-   foreach (IBlueRow r in dp.addedRows) {
-      for (int i = 0; i < colWidths.Length; i++) {
-         var str = r[i]?.ToString() ?? string.Empty;
-         if (i > 0) {
-            sb.Append(" | ");
-         }
-         sb.Append(str);
-         var width = colWidths[i];
-         if (str.Length < width) {
-            sb.Append(new string(' ', width - str.Length));
-         }
-      }
-      sb.AppendLine();
-   }
-   return sb.ToString();
-}
-
-var parser = new Parser(ps => {
-   ps.CaseInsensitiveEnumValues = true;
-   ps.HelpWriter = Console.Out;
-});
-
-await parser.ParseArguments<DataStreamWebAppServerOptions>(args).
-   MapResult(run, errors => Task.FromResult(1));
-
-async Task<int> run(DataStreamWebAppServerOptions opts) {
-   BTSNativeMethods.provide_command_line(args, args.Length);
-   Configuration.initialize();
-   var context = ServiceContext.create(opts.sparkd);
-   await context.login(opts.user!, opts.password!);
-   var vsWorkLoop = Observable.Interval(TimeSpan.FromMilliseconds(200)).Subscribe(_ => {
-      ViewStreamDatabase.instance.doWork();
-      MasterWorker.getInstance(false).doWork();
-   });
-   var builder = WebApplication.CreateBuilder(args);
-   builder.Services.AddDirectoryBrowser();
-   var app = builder.Build();
-   var wsOpts = new WebSocketOptions {
-      KeepAliveInterval = TimeSpan.FromMinutes(2)
-   };
-   app.UseWebSockets(wsOpts);
-   app.UseDefaultFiles();
-   app.UseStaticFiles();
-   app.MapFallbackToFile("/index.html");
-
-   app.Use(async (context, next) => {
-      var reqDesc = fmtReq(context.Request);
-      Console.WriteLine($"Webserver received: {reqDesc}");
-      if (context.Request.Path == "/ws") {
-         if (context.WebSockets.IsWebSocketRequest) {
-            Console.WriteLine("Accepting websocket connection...");
-            using var webSocket = await context.WebSockets.AcceptWebSocketAsync();
-            Console.WriteLine("Websocket connection accepted.");
-            await runWebSocketServer(
-               opts.sparkd, opts.user!, opts.password!, webSocket);
-         } else {
-            context.Response.StatusCode = StatusCodes.Status400BadRequest;
-         }
-      } else {
-         Console.WriteLine($"Webserver ignoring req: {reqDesc}");
-         await next(context);
-      }
-   });
-
-   try {
-      await app.RunAsync();
-   } catch (Exception ex) {
-      Console.WriteLine($"Exception on DataStreamWebsocketServer thread: {ex}");
-      return 1;
-   }
-   Spark.Core.Disposables.Disposable.dispose(ref vsWorkLoop);
-   return 0;
-}
-
-async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
-   var buffer = new byte[1024 * 4];
-   var receiveResult = await webSocket.ReceiveAsync(
-       new ArraySegment<byte>(buffer), CancellationToken.None);
-
-   try {
-      while (!receiveResult.CloseStatus.HasValue) {
-         Console.WriteLine("Awaiting websocket message from client");
-         receiveResult = await webSocket.ReceiveAsync(
-             new ArraySegment<byte>(buffer), CancellationToken.None);
-         var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
-         var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
-         var queryArgs = rpcReq!.Params!;
-         // var queryArgs = new DataStreamSubscription { Query = msg };
-         var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
-         if (exp is null) {
-            Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
-            continue;
-         }
-         var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
-         Console.WriteLine("Subscribing to stream...");
-         vs.onChange += onStreamChanged;
-
-         await webSocket.SendAsync(
-             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-      }
-   } catch (Exception ex) {
-      Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
-      return 1;
-   }
-
-   return 0;
-
-   async void onStreamChanged(object newValue) {
-      if (newValue is not DataPacket dp) {
-         Console.WriteLine(
-            $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
-         return;
-      }
-      Console.WriteLine("Received stream update.");
-      var sb = new StringBuilder();
-      var updateDesc = fmtDataPacket(dp);
-      var msg = $"Received new rows:\n{updateDesc}";
-      // var payload = new ReadOnlyMemory<byte>([.. msg]);
-      var encoded = Encoding.UTF8.GetBytes(msg);
-      var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
-      await webSocket.SendAsync(
-          payload,
-          receiveResult.MessageType,
-          receiveResult.EndOfMessage,
-          CancellationToken.None);
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -1,48 +0,0 @@
-{
-  "profiles": {
-    "default": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "applicationUrl": "http://localhost:5400"
-    },
-    "http": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      },
-      "dotnetRunMessages": true,
-      "applicationUrl": "http://localhost:5268"
-    },
-    "https": {
-      "commandName": "Project",
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      },
-      "dotnetRunMessages": true,
-      "applicationUrl": "https://localhost:7128;http://localhost:5268"
-    },
-    "IIS Express": {
-      "commandName": "IISExpress",
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      }
-    }
-  },
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "iisSettings": {
-    "windowsAuthentication": false,
-    "anonymousAuthentication": true,
-    "iisExpress": {
-      "applicationUrl": "http://localhost:2083",
-      "sslPort": 44383
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WeatherForecast.cs

@@ -1,12 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class WeatherForecast
-{
-   public DateOnly Date { get; set; }
-
-   public int TemperatureC { get; set; }
-
-   public int TemperatureF => 32 + (int)(TemperatureC / 0.5556);
-
-   public string? Summary { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WebsocketConnectionManager.cs

@@ -1,37 +0,0 @@
-using System;
-using System.Collections.Concurrent;
-using System.Net.WebSockets;
-
-namespace Bts.DataStreamWebApp.Server {
-   // public class WebSocketConnectionManager {
-   //    private readonly ConcurrentDictionary<Guid, WebSocket> _sockets = new ConcurrentDictionary<Guid, WebSocket>();
-
-   //    public Guid AddSocket(WebSocket socket) {
-   //       var socketId = Guid.NewGuid();
-   //       _sockets.TryAdd(socketId, socket);
-   //       return socketId;
-   //    }
-
-   //    public WebSocket? GetSocket(Guid socketId) {
-   //       _sockets.TryGetValue(socketId, out var socket);
-   //       return socket;
-   //    }
-
-   //    public ConcurrentDictionary<Guid, WebSocket>.ValueCollection GetAllSockets() {
-   //       return _sockets.Values;
-   //    }
-
-   //    public Guid? GetSocketId(WebSocket socket) {
-   //       foreach (var (key, value) in _sockets) {
-   //          if (value == socket) {
-   //             return key;
-   //          }
-   //       }
-   //       return null;
-   //    }
-
-   //    public void RemoveSocket(Guid socketId) {
-   //       _sockets.TryRemove(socketId, out _);
-   //    }
-   // }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -1,8 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -1,9 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*"
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -1,168 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>ALT WebSocket Sample Application</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Message to send:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            var data = sendMessage.value;
-            socket.send(data);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -1,176 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>DataStreams via WebSocket</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Query:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            const rpcReq = {
-              jsonrpc: "2.0",
-              method: "subscribe",
-              params: {
-                query: sendMessage.value,
-              },
-              id: 1,
-            };
-            const rpcReqStr = JSON.stringify(rpcReq);
-            socket.send(rpcReqStr);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/DataStream/Misc/PandasScriptStream.cs

@@ -0,0 +1,29 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+using System.IO;
+
+public class PandasScriptStream : ViewStream {
+   public PandasScriptStream(ViewStream vs, string file)
+      : this(vs, file, null) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
+      : this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
+      mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
+
+      string readFile() {
+         try {
+            return File.ReadAllText(file);
+         } catch (Exception ex) {
+            Log.Gen.error($"PandasScriptStream error reading file: {ex}");
+            throw;
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,62 +1,130 @@
-// using System;
-// using System.Diagnostics;
-
-// using BCG;
-// using BCG.Utilities.General;
-
-
-// namespace bts {
-
-//     public class PandasStream : ViewStream, IWorker {
-//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
-
-//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
-
-//         }
-
-//         public PandasStream(ViewStream vs, string pandas_code) :
-//         this(vs, ViewStreamDatabase.instance, pandas_code) {
-//         }
-
-
-//         protected override void myOnFirstSubscriber() {
-
-//             // string python_code = @"
-//             //    import winsound
-//             //    frequency = 2500  # Set frequency (in Hz)
-//             //    duration = 1000   # Set duration (in milliseconds)
-//             //    winsound.Beep(frequency, duration)
-//             // ";
-//             string python_code = @"
-//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
-//             with open(file_path, \"w\") as f:
-//                 f.write(\"Hello, world!\n\")
-//             ";
-
-//             execute_python_code(python_code);
-//         }
-
-//         static void execute_python_code(string python_code) {
-//             ProcessStartInfo start = new ProcessStartInfo();
-//             start.FileName = "python"; // Python executable
-//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
-//             start.UseShellExecute = false;
-//             start.RedirectStandardOutput = true;
-
-//             using (Process process = Process.Start(start)) {
-//                 using (System.IO.StreamReader reader = process.StandardOutput) {
-//                     string result = reader.ReadToEnd();
-//                     Console.Write(result);
-//                 }
-//             }
-//         }
-
-//         protected override void myOnLastSubscriber() {
-//         }
-
-//         public void doWork() {
-
-//         }
-
-//     }
-// }
+namespace Bts.DataStream;
+
+using BCG;
+using BCG.Extensions;
+using BCG.Utilities.General;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+public class PandasStream : ViewStream, IWorker {
+   readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
+   readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
+   readonly string pythonExe;
+   readonly ViewStreamDatabase vsd;
+   readonly CSVSaverStream inputCsvStream;
+   readonly Lazy<string> pythonCode;
+   IDictionary<int, IBlueRow>? lastRowMap = null;
+   int inputVersion = 0;
+   int outputVersion = 0;
+   DateTime lastFireTime = DateTime.MinValue;
+   TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
+   IDisposable? dispose;
+
+   public PandasStream(ViewStream vs, string dataFrameExpression)
+      : this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
+
+   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
+      pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
+      this.pythonExe = pythonExe;
+      this.vsd = vsd;
+      // This should fire an update when it writes to the output file
+      inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
+      pythonCode = new(getPythonCode);
+
+      string getPythonCode() {
+         var imports =
+@"import numpy as np
+import pandas as pd
+import sys
+";
+         var bd = new StringBuilder(imports);
+         bd.AppendLine($"df=pd.read_csv('{inputFile}')");
+         bd.AppendLine(dataFrameExpression);
+         bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
+         return bd.ToString();
+      }
+   }
+
+   protected override void myOnFirstSubscriber() {
+      vsd.addWorker(this);
+      inputCsvStream.onChange += onInputChanged;
+      this.dispose = Disposable.Create(dispose);
+
+      void onInputChanged(object newValue) =>
+         inputVersion++;
+
+      void dispose() {
+         inputCsvStream.onChange -= onInputChanged;
+         vsd.removeWorker(this);
+         deleteFile(inputFile);
+         deleteFile(outputFile);
+      }
+
+      void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
+         if (!File.Exists(file)) {
+            return;
+         }
+         try {
+            File.Delete(file);
+         } catch (Exception ex) {
+            Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref dispose);
+
+   static void executePythonCode(string pythonCode, string? pythonExe = null) {
+      pythonExe ??= "python";
+      var start = new ProcessStartInfo {
+         FileName = pythonExe,
+         Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
+         UseShellExecute = false,
+         RedirectStandardOutput = true
+      };
+      using var process = Process.Start(start)
+         ?? throw new InvalidOperationException("Failed to launch python");
+      process.WaitForExit();
+   }
+
+   void IWorker.doWork() {
+      if (outputVersion == inputVersion) {
+         return;
+      }
+      var now = DateTime.Now;
+      var elapsedSinceLastFire = now - lastFireTime;
+      if (elapsedSinceLastFire < fireThrottle) {
+         return;
+      }
+      outputVersion = inputVersion;
+      executePythonCode(pythonCode.Value, pythonExe);
+      var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
+      var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
+      var comp = BlueRowEqualityComparer.Instance;
+      var deletedRows = new BlueTable(schema);
+      var addedRows = new BlueTable(schema);
+      if (lastRowMap is not null) {
+         deletedRows.SetItems(lastRowMap.Where(oldRow =>
+            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
+         addedRows.SetItems(rowMap.Where(newRow =>
+            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
+      }
+      lastRowMap = rowMap;
+      lastFireTime = DateTime.Now;
+      myFireEvent(new DataPacket(rows, addedRows, deletedRows));
+   }
+}

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -14,8 +14,11 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
       this.source = source;
 
    public void Dispose() {
-      subscription?.Dispose();
-      subscription = null;
+      GC.SuppressFinalize(this);
+      if (subscription is not null) {
+         subscription.Dispose();
+         subscription = null;
+      }
    }
 
    protected override void myOnFirstSubscriber() {

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -1,32 +0,0 @@
-using BCG;
-
-namespace BCG
-{
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("add_to_cloud")]
-   public class AddToCloudStream : ViewStream
-   {
-      private readonly string _name, _stream_exp;
-      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
-      public AddToCloudStream(string name, params string[] lines)
-      : this(name, 0, lines)
-      {
-      }
-
-      public AddToCloudStream(string name, int persist, params string[] lines)
-      {
-         _name = name;
-         _stream_exp = getExpression(lines);
-         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
-         mySetRealSource(new TableStream("cloud_streams"));
-      }
-
-      static string getExpression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,31 +1,22 @@
-using BCG;
-
 using System;
 using System.Collections.Generic;
 
-using mbts.config;
-using bts.utils;
-using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
-using System.Collections.Generic;
 using BCG.Utilities.General;
+using bts;
 
-namespace BCG
-{
+namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class CloudStreamManager : IWorker
-   {
-      public class StreamPubUnit
-      {
+   public class RemoteQueryManager : IWorker {
+      public class StreamPubUnit {
          public string name;
          public string stream_expression;
          public ViewStream stream;
          public bool persist;
          public ViewPublisher view_publisher;
 
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
-         {
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
             name = name_;
             stream_expression = stream_expression_;
             stream = stream_;
@@ -33,7 +24,7 @@ namespace BCG
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream build_stream(string stream_expression)
+      static ViewStream? build_stream(string stream_expression)
       {
          try
          {
@@ -41,9 +32,7 @@ namespace BCG
             var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
             bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
@@ -52,34 +41,37 @@ namespace BCG
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       private ViewStream _stream = new TableStream("cloud_streams");
       private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
-      public CloudStreamManager()
-      {
+
+      public RemoteQueryManager() {
          _stream.onChange += _on_str;
-
+         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
       }
       private void _on_str(object obj)
       {
-         DataPacket dp = obj as DataPacket;
+         if (obj is not DataPacket dp) {
+            return;
+         }
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
             bool persist = r.get<bool>("persist");
             string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name))
-            {
-               _assign(name, stream_exp, persist);
+            if (!_stream_pub_map.ContainsKey(name)) {
+               _spawn_remote_stream(name, stream_exp, persist);
             }
          }
       }
 
-      private void _assign(string name, string stream_expression, bool persist)
-      {
+      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
-         if (persist)
-         {
+         if (persist) {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -87,14 +79,12 @@ namespace BCG
          _stream_pub_map.Add(name, u);
       }
 
-      private void _expose_to_ws(string name, ViewStream stream)
-      {
+      private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
 
-      public void doWork()
-      {
+      public void doWork() {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -0,0 +1,48 @@
+using BCG;
+
+using bts;
+
+using System;
+
+namespace BCG {
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
+   public class RemoteQueryStream : ViewStream {
+      private readonly string _name, _stream_exp;
+      private PublisherCore _publisher_core = new PublisherCore();
+      private ViewStream _source_str;
+
+      public RemoteQueryStream(params string[] lines) {
+         _stream_exp = get_expression(lines);
+         _name = Guid.NewGuid().ToString();
+         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
+         //_name.Replace("-", "_");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
+         byte[] rem_req_bytes = rem_req.serialize();
+         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
+      }
+
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }
+
+      static string get_expression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+
+      private void _handle_data(object o) {
+         myFireEvent(o);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -0,0 +1,31 @@
+namespace bts {
+   using System.IO;
+   using BCG;
+
+   using System.IO;
+   using System.Threading.Tasks.Dataflow;
+
+   public class RemoteRequest {
+      public readonly string stream_expression;
+      public readonly string published_table_name;
+      public RemoteRequest(string stream_expression_, string published_table_name_) {
+         stream_expression = stream_expression_;
+         published_table_name = published_table_name_;
+      }
+      public byte[] serialize() {
+         var str = new MemoryStream();
+         var bw = new BinaryWriter(str);
+         bw.write_string_cpp(stream_expression);
+         bw.write_string_cpp(published_table_name);
+         return str.ToArray();
+      }
+
+      public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
+         BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
+         string stream_expression_ = br.read_string_cpp();
+         string published_table_name_ = br.read_string_cpp();
+         return new RemoteRequest(stream_expression_, published_table_name_);
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -1,59 +1,54 @@
-using BCG;
+// using BCG;
 
-using System.Linq;
+// using bts;
 
-namespace BCG
-{
-   public class StreamOffloadRequestor
-   {
-      public class OffloadOp
-      {
-         string name;
-         string stream_exp;
-      }
-      private IBlueTableSchema _schema;
-      private IBlueRowCollection _rows;
-      private PubTableBroker _broker = new PubTableBroker();
-      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-      public StreamOffloadRequestor()
-      {
-         _schema = new BlueTableSchema("cloud_streams",
-                                   new ColumnInfo("id", 0, typeof(int)),
-                                   new ColumnInfo("name", 1, typeof(string)),
-                                   new ColumnInfo("persist", 2, typeof(bool)),
-                                   new ColumnInfo("stream_exp", 3, typeof(string)));
-         _existing_offloads_vs.onChange += _on_str;
-      }
-      private void _on_str(object obj)
-      {
-         DataPacket dp = obj as DataPacket;
-         _rows = dp.newRows;
-         // this is needed so the broker knows the state of the table
-         _broker.createTable(_rows);
-      }
-      public void add_to_cloud(string name, string stream_exp, bool persist)
-      {
-         // check if it exists
-         if (_rows == null)
-         {
-            BlueTable table = new BlueTable(_schema);
-            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-            table.addRow(nr);
-            _broker.createTable(table);
-            // create table  
-         }
-         else
-         {
-            if (exists(name))
-               return;
-            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-            _broker.addRow(nr);
-         }
-      }
+// using System.Linq;
 
-      bool exists(string name)
-      {
-         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-      }
-   }
-}
+// namespace BCG {
+//    // used on the client
+//    public class StreamOffloadRequestor {
+//       public class OffloadOp {
+//          string name;
+//          string stream_exp;
+//       }
+//       private IBlueTableSchema _schema;
+//       private IBlueRowCollection _rows;
+//       private PubTableBroker _broker = new PubTableBroker();
+//       ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+//       public StreamOffloadRequestor() {
+//          _schema = new BlueTableSchema("cloud_streams",
+//                                    new ColumnInfo("id", 0, typeof(int)),
+//                                    new ColumnInfo("name", 1, typeof(string)),
+//                                    new ColumnInfo("persist", 2, typeof(bool)),
+//                                    new ColumnInfo("stream_exp", 3, typeof(string)));
+//          _existing_offloads_vs.onChange += _on_str;
+//       }
+//       private void _on_str(object obj) {
+//          DataPacket dp = obj as DataPacket;
+//          _rows = dp.newRows;
+//          // this is needed so the broker knows the state of the table
+//          _broker.create_table(_rows);
+//       }
+//       public void add_to_cloud(string name, string stream_exp, bool persist) {
+//          // check if it exists
+//          if (_rows == null) {
+//             BlueTable table = new BlueTable(_schema);
+//             IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+//             table.addRow(nr);
+//             _broker.create_table(table);
+//             // create table  
+//          } else {
+//             if (exists(name))
+//                return;
+//             IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+//             // TODO: use make_batch_changes instead
+//             Log.assert(false, "Fix addRow");
+//             //_broker.addRow(nr);
+//          }
+//       }
+
+//       bool exists(string name) {
+//          return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs

@@ -1,17 +1,14 @@
 using System;
 using System.Collections;
 
-namespace BCG
-{
-   internal class AddColumnsRow : IBlueRow
-   {
+namespace BCG {
+   internal class AddColumnsRow : IBlueRow {
       private IBlueTableSchema mySchema;
       private IBlueRow myRow;
       //private object[] myCachedValues;
 
       #region Constructors
-      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
-      {
+      public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
          mySchema = schema;
          myRow = row;
          //myCachedValues = new object[schema.columnNames.Length];
@@ -19,85 +16,78 @@ namespace BCG
       #endregion
 
       #region IBlueRow members
-      public IBlueTableSchema schema{get{return mySchema;}}
+      public IBlueTableSchema schema { get { return mySchema; } }
       #endregion
 
       #region ICollection members
-      public int Count {get{return mySchema.columnNames.Length;}}
+      public int Count { get { return mySchema.columnNames.Length; } }
 
-      public bool IsSynchronized {get{return false;}}
+      public bool IsSynchronized { get { return false; } }
 
-      public object SyncRoot {get{return this;}}
+      public object SyncRoot { get { return this; } }
 
-      public void CopyTo (Array array, int index)  
-      {
+      public void CopyTo(Array array, int index) {
          int i = index;
          foreach (object o in this as IEnumerable)
-            array.SetValue (o, i++);
+            array.SetValue(o, i++);
       }
       #endregion
 
       #region IDictionary members
-      public bool IsFixedSize {get{return true;}}
+      public bool IsFixedSize { get { return true; } }
 
-      public bool IsReadOnly {get{return true;}}
+      public bool IsReadOnly { get { return true; } }
 
       /// <summary>
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public object this [object key]
-      {
-         get
-         {
+      public object this[object key] {
+         get {
             if (key is string)
                return this[(string)key];
             return this[(int)key];
          }
-         set {throw new NotSupportedException ("The IBlueRow is readonly");}
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
       }
 
-      public object this[int index]
-      {
-         get
-         {
+      public object this[int index] {
+         get {
             //if (myCachedValues[index] == null)
             //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
             //return myCachedValues[index];
-            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+            return mySchema[index].isConstant ? DBNull.Value : myRow[index];
          }
       }
 
-      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}
+      public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
 
-      public ICollection Keys{get{return mySchema.columnNames;}}
+      public ICollection Keys { get { return mySchema.columnNames; } }
 
-      public ICollection Values
-      {
-         get
-         {
+      public ICollection Values {
+         get {
             object[] retVal = new object[mySchema.columnNames.Length];
-            CopyTo (retVal, 0);
+            CopyTo(retVal, 0);
             return retVal;
          }
       }
 
-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
 
-      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
 
-      public bool Contains (object key){return mySchema[(string)key] != null;}
+      public bool Contains(object key) { return mySchema[(string)key] != null; }
 
       /// <summary>
       /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
       /// iterate through the values, typecast this into an IEnumerable
       /// </summary>
       /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
-      
-      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
 
-      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
       #endregion
 
    }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,6 +1,34 @@
-namespace BCG;
-
-public class AddColumnsStream : MapStream {
-   public AddColumnsStream(ViewStream vs, string columnDef)
-      : base(vs, new ColumnAdder(columnDef)) { }
-}
+using System;
+
+namespace BCG {
+   public class AddColumnsStream : ViewStream {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
+         string[] toks = exp.Split(':');
+         string col_name = toks[0];
+         string col_value = toks[1];
+         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
+         return Tuple.Create(col_name, tup.Item1, tup.Item2);
+      }
+
+      private Tuple<string, Type, object> _new_col;
+      private IBlueTableSchema _out_schema;
+
+      public AddColumnsStream(ViewStream vs, string col_def) {
+         _new_col = parse_exp(col_def);
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
+         mySetRealSource(rs);
+      }
+
+      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
+         return _out_schema;
+      }
+
+      private IBlueRow _map(IBlueRow r) {
+         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
+         rv.setValue(_new_col.Item1, _new_col.Item3);
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,48 +1,24 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
+using BCG;
 
-using System.Runtime.Serialization.Formatters.Binary;
-
-namespace BCG
-{
+namespace bts {
    [DataStreamUserRespresentation("loader")]
-   public class LoaderStream : ViewStream, ISideEffectStream
-   {
-      public class Exception : System.Exception
-      {public Exception(string msg) : base(msg) { }}
+   public class LoaderStream : ViewStream {
 
-      public LoaderStream(string fileName, int secondsBetweenPolls)
-      {
-         mySetRealSource(
-            new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-               ViewStreamDatabase.instance));
+      private string _filename;
+      private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
+
+
+      public LoaderStream(string filename) {
+         _filename = filename;
       }
 
-      public LoaderStream(string fileName)
-      {
-        if (fileName == null)
-           throw new LoaderStream.Exception("File name cannot be null");
-
-        for (int i = 0; i < 5; i++)
-           try
-           {
-              using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-              {
-#pragma warning disable SYSLIB0011
-                  BinaryFormatter bf = new BinaryFormatter();
-                  BlueTable bt = (BlueTable)bf.Deserialize(fs);
-#pragma warning restore SYSLIB0011
-                  mySetRealSource(new BlueRowCollectionStaticStream(bt));
-              }
-              break;
-           }
-           catch (IOException)
-           {
-              System.Threading.Thread.Sleep(50);
-           }
-
+      protected override void myOnFirstSubscriber() {
+         byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
+         StreamIOMessage msg = _deser.deserialize(ser_msg);
+         Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
+         IBlueRowCollection rows = msg.create_message.rows;
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,6 +5,7 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
+using bts;
 
 namespace BCG {
    // Writes all the udpates to a stream in a file.
@@ -24,7 +25,7 @@ namespace BCG {
       private ViewStream _source_stream;
       private string _token;
       private FileStream _out_fs;
-      private IBlueDataSerializer _serializer;
+      private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
       private string _file_name = String.Empty;
       private long _rollover_size;
       private readonly string _directory_path;
@@ -94,13 +95,12 @@ namespace BCG {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
          bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
+         _serializer = _serializer ?? new UniversalStreamIOSerializer();
          if (_write_create) {
             // new files always starts with create
-            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
-            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
-
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
          BinaryWriter bw = new BinaryWriter(_out_fs);
          long utc_epoch = epoch_now();
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +1,68 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
 
-namespace BCG
-{
-   public class RecorderStream : ViewStream
-   {
-      private ViewStream mySource;
-      private DateTime myStartTime;
-      private FileStream myOutFile;
-      private string myFileName;
-      private IBlueDataSerializer mySerializer;
-            
-      public RecorderStream(ViewStream src, string recordFile)
-      {
-         mySource = src;
-         myFileName = recordFile;
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;
 
-         myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-      }
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;
 
-      protected override void myOnFirstSubscriber()
-      {
-         myStartTime = DateTime.Now;         
-         mySource.onChange += new ChangeDelegate(mySource_onChange);
-         myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-      }
-      
-      protected override void myOnLastSubscriber()
-      {
-         mySource.onChange -= mySource_onChange;
-         myOutFile.Close();
-      }
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }
 
-      void mySource_onChange(object newValue)
-      {
-         DataPacket dp = (DataPacket)newValue;         
-         
-         BlueTable toWrite = new BlueTable(dp.schema);
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }
 
-         if (mySerializer == null)
-         {
-            mySerializer = new UniversalSerializer(dp.schema);
-         }
-         foreach (IBlueRow r in dp.newRows)
-            toWrite.addRow(r);
-         
-         byte[] result;
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }
 
-         if (myLastValue == null)
-            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-         else
-            mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         
 
-         BinaryWriter bw = new BinaryWriter(myOutFile);
-         //write down what time this happened
-         bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-         bw.Write(result.Length);
-         bw.Write(result);         
-         myOutFile.Flush();
+//          BlueTable toWrite = new BlueTable(dp.schema);
 
-         myFireEvent(dp);
-      }
-   }
-}
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -1,64 +1,37 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using BCG;
 
-// using System.Runtime.Serialization.Formatters.Binary;
+namespace bts {
+   // this saves the total state in the filr provided in the cons
+   [DataStreamUserRespresentation("saver")]
+   public class SaverStream : ViewStream {
+      private ViewStream _stream;
+      private string _filename;
+      private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
 
-// namespace BCG
-// {
-//    [DataStreamUserRespresentation("saver")]
-//    public class SaverStream : ViewStream, ISideEffectStream
-//    {
-//       private ViewStream myStream;
-//       private string myFileName;
+      public SaverStream(ViewStream vs, string filename) {
+         _stream = vs;
+         _filename = filename;
+      }
 
-//       public class Exception : System.Exception
-//       {
-//          public Exception(string msg) : base(msg) { }
-//       }
+      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
 
-//       public SaverStream(ViewStream vs, string fileName)
-//       {
-//          if (fileName == null)
-//             throw new SaverStream.Exception("File name cannot be null");
-//          myStream = vs;
-//          myFileName = fileName;
+      void _handle_changes(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _save(dp);
+         myFireEvent(dp_obj);
+      }
 
-//       }
+      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
 
-//       protected override void myOnFirstSubscriber()
-//       { myStream.onChange += new ChangeDelegate(myHandleChanges); }
+      protected void _save(DataPacket dp) {
+         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
+         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
+         StreamSerializationUtils.write_to_file(str_bytes, _filename);
 
-//       void myHandleChanges(object newValue)
-//       {
-//          DataPacket dp = newValue as DataPacket;
-//          mySave(dp);
-//          myFireEvent(newValue);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
-
-//       protected void mySave(DataPacket dp)
-//       {
-//          for (int i = 0; i < 5 ; i++)
-//             try
-//             {
-//                using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
-//                {
-//                   BinaryFormatter bf = new BinaryFormatter();
-//                   BlueTable t = new BlueTable(dp.schema);
-//                   foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
-//                      t.addRow(new BlueRow(r));
-//                   bf.Serialize(fs, t);
-//                }
-//                break;
-//             }
-//             catch (IOException)
-//             {
-//                System.Threading.Thread.Sleep(5);
-//             }
-//       }
-//    }
-// }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -1,12 +1,12 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using System.Runtime.Serialization.Formatters.Binary;
-using BCG.Utilities.General;
-using bts.collections;
-using System.Reflection.Metadata;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
 
 
 namespace BCG {
@@ -15,115 +15,115 @@ namespace BCG {
          FirstOfDay,
          LastOfDay
       };
-      private BlueTable _current_table;
-      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-      public DateTime _snapshot_time;
-      public string _snapshot_file;
-      public IEnumerable<PlaybackPacket> _packets;
-      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-         DateTime dt;
-         var parsed = DateTime.TryParse(snapshot_date, out dt);
-         dt = dt.Date;
-         bts.Log.assert(parsed);
-         SpecificSnapshot specific_snapshot_enum;
-         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-         bts.Log.assert(parsed);
-         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-         mySetRealSource(rs);
-      }
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       public DateTime _snapshot_time;
+//       public string _snapshot_file;
+//       public IEnumerable<PlaybackPacket> _packets;
+//       public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+//          DateTime dt;
+//          var parsed = DateTime.TryParse(snapshot_date, out dt);
+//          dt = dt.Date;
+//          bts.Log.assert(parsed);
+//          SpecificSnapshot specific_snapshot_enum;
+//          parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+//          bts.Log.assert(parsed);
+//          string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+//          ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+//          mySetRealSource(rs);
+//       }
 
-      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-         string token_directory_name = $"{directory_name}\\{token}";
-         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-      }
+//       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+//          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+//       }
 
-      protected override void myOnFirstSubscriber() {
-         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-         _packets = StreamPersistorUtils.load_packets_from_file(fs);
-         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-         if (snapshot_packet != null) {
-            foreach (PlaybackPacket p in _packets) {
-               byte[] ops;
-               Array row_array;
-               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-               for (int i = 0; i < ops.Length; i++) {
-                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-                  byte op = ops[i];
-                  if (op == StreamOpCode.create) {
-                     IBlueRowCollection full_table = rows_args[0];
-                     _current_table = new BlueTable(full_table);
-                  } else if (op == StreamOpCode.multipleRemoveAdd) {
-                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
-                  } else {
-                     bts.Log.assert(false, $"Oprator: {op} is not expected");
-                  }
-                  if (p.idx == snapshot_packet.idx) {
-                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-                     break;
-                  }
-               }
-            }
-         }
-      }
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+//          _packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
 
-      private void _absorb_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-            } else {
-               bts.Log.assert(false, $"Operator: {op} is not expected");
-            }
-         }
-      }
+//       private void _absorb_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//             } else {
+//                bts.Log.assert(false, $"Operator: {op} is not expected");
+//             }
+//          }
+//       }
 
-      protected override void myOnLastSubscriber() {
-      }
+//       protected override void myOnLastSubscriber() {
+//       }
 
-      // public event SnapshotStream.Progress onProgress;
-      // public event Action onFileExhausted;
-      private PlaybackPacket _locate_packet_to_replay() {
-         foreach (var p in _packets) {
-            // returns the first non replayed packet
-            if (!p.replayed)
-               return p;
-         }
-         return null;
-      }
+//       // public event SnapshotStream.Progress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          return null;
+//       }
 
-      private void _fire_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-               //_current_table.addRows(full_table);
-               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-                                          (myLastValue as DataPacket).newRows));
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-               myFireEvent(new DataPacket(_current_table, adds, dels));
-            } else {
-               bts.Log.assert(false, $"Oprator: {op} is not expected");
-            }
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
 
-         }
-      }
+//          }
+//       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,16 +1,10 @@
-using BCG;
-using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using System.Text.RegularExpressions;
-
 using Bts.DataStream;
 
-using mbts;
-using mbts.spark;
 using mbts.config;
 
+using System;
+using System.Text.RegularExpressions;
+
 namespace BCG.StreamWorld.Addition
 {
    [DataStreamUserRespresentation (false)]
@@ -151,7 +145,6 @@ namespace BCG
       private string myFilterString;
       private BCG.StreamWorld.Addition.FilterStream myFilterStream;
       private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -226,7 +219,6 @@ namespace BCG
 
       private void myHandleCallbacks(object newValue)
       {
-         _timer.start();
          DataPacket retVal;
          DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
@@ -255,7 +247,6 @@ namespace BCG
                if (myFilter.filter(r))
                   myLastFiredRows.addRow(r);
          }
-         _timer.stop();
          myFireEvent(retVal);
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -1,37 +1,66 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using System.Text.RegularExpressions;
 
 using BCG.Utilities.General;
 
-namespace BCG
-{
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
-   {
+namespace BCG {
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
+
+      public static TimeSpan parse_duration(string input) {
+         int days = 0, hours = 0, minutes = 0, seconds = 0;
+
+         // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
+         Regex regex = new Regex(@"(\d+)([dhms])");
+         MatchCollection matches = regex.Matches(input);
+
+         foreach (Match match in matches) {
+            int value = int.Parse(match.Groups[1].Value);
+            string unit = match.Groups[2].Value;
+
+            switch (unit) {
+               case "d":
+                  days = value;
+                  break;
+               case "h":
+                  hours = value;
+                  break;
+               case "m":
+                  minutes = value;
+                  break;
+               case "s":
+                  seconds = value;
+                  break;
+            }
+         }
+
+         return new TimeSpan(days, hours, minutes, seconds);
+      }
+
       #region constructors
+
+      public TimeBasedFilter(string col_name, string row_time_to_live)
+      : this(col_name, parse_duration(row_time_to_live)) {
+
+      }
+
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-      {
-         myVsd               = vsd;
-         myRowTimeColumnName = rowTimeColumnName;
-         myRowTimeToLive     = rowTimeToLive;
-         myFilterFrequency   = filterFrequency;
-         myNextFilter        = DateTime.MinValue;
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+         myVsd = vsd;
+         _time_col = rowTimeColumnName;
+         _ttl = rowTimeToLive;
+         myFilterFrequency = filterFrequency;
+         myNextFilter = DateTime.MinValue;
 
          // add worker
          myVsd.addWorker(this);
@@ -41,32 +70,25 @@ namespace BCG
       public event FilterChangeDelegate onFilterChange;
 
       #region IStreamFilter
-      public bool filter(IBlueRow row)
-      {
-         DateTime now       = DateTime.Now;
-         DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
+      public bool filter(IBlueRow row) {
+         DateTime now = DateTime.Now;
 
-         if (now > timeStamp)
-         {
-            TimeSpan span = now - timeStamp;
-
-            if (span >= myRowTimeToLive)
-            {
+         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
+         if (now > row_time) {
+            TimeSpan span = now - row_time;
+            if (span >= _ttl) {
                return (false);
             }
          }
-
          return (true);
       }
       #endregion
 
       #region IWorker
-      public void doWork()
-      {
+      public void doWork() {
          DateTime now = DateTime.Now;
 
-         if (now > myNextFilter)
-         {
+         if (now > myNextFilter) {
             // fire event
             onFilterChange();
 
@@ -76,8 +98,7 @@ namespace BCG
       #endregion
 
       #region IDisposable
-      public void Dispose()
-      {
+      public void Dispose() {
          // remove worker
          myVsd.removeWorker(this);
       }
@@ -85,10 +106,20 @@ namespace BCG
 
       #region state
       private ViewStreamDatabase myVsd;
-      private string             myRowTimeColumnName;
-      private TimeSpan           myRowTimeToLive;
-      private TimeSpan           myFilterFrequency;
-      private DateTime           myNextFilter;
+      private string _time_col;
+      private TimeSpan _ttl;
+      private TimeSpan myFilterFrequency;
+      private DateTime myNextFilter;
       #endregion
    }
+   [DataStreamUserRespresentation("timefilter")]
+   public class TimeBasedFilterStream : ViewStream {
+      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
+      :
+      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
+      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
+         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
+         mySetRealSource(new FilterStream(base_stream, filter));
+      }
+   }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -4,18 +4,14 @@ using BCG.Utilities;
 using System.Collections.Generic;
 using System.Linq;
 
-namespace BCG
-{
-   public class GroupByStream : MethodDrivenStream
-   {
-      private class MyFuncGroupByEvaluator : IGroupByEvaluator
-      {
+namespace BCG {
+   public class GroupByStream : MethodDrivenStream {
+      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         {
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
-            _groupby_func = groupby_func;            
+            _groupby_func = groupby_func;
          }
          public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
 
@@ -29,7 +25,7 @@ namespace BCG
       /// Serves as  a quick way to access all rows belonging to the same group
       /// </summary>
       private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable ;
+      private RowDictionaryTable myBookKeepingTable;
       private IGroupByEvaluator myGroupByEvaluator;
       private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
       //can map a schema to another
@@ -37,112 +33,98 @@ namespace BCG
       private int myIdGenerator = 0;
 
       public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
-      { }
+         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
-         : this (stream, groupByEvaluator, true)
-      {}
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
+         : this(stream, groupByEvaluator, true) { }
 
-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
-         : base (stream)
-      {
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
+         : base(stream) {
          myGroupByEvaluator = groupbyEvaluator;
          myIdGuard = idGuard;
       }
 
-      protected override void myHandleFirstCall ()
-      {
+      protected override void myHandleFirstCall() {
          mySchema = myLastArrivedPacket.schema;
-         myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
-         myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, 
+         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
+         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
             myGroupByEvaluator.groupByColumns);
-         GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
-         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
-         {
-            IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
-               rowGroup, new BlueTable (mySchema)));
+         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
+         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
+            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
+               rowGroup, new BlueTable(mySchema)));
             if (newRow != null)
-               temp.addRow (newRow, myGetGroupString (rowGroup));
+               temp.addRow(newRow, myGetGroupString(rowGroup));
          }
-         myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
+         myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
       }
 
       /// <summary>
       /// Adding need not to anything as deletion already takes care
       /// of addition
       /// </summary>
-      protected override void myHandleIncrementalAdd()   {}
-      
+      protected override void myHandleIncrementalAdd() { }
+
       /// <summary>
       /// Both deletes and adds are handled here to make the intermediate results not
       /// appear in the spitted fired DataPacket (note this is done only in GroupByStream 
       /// because of the nature of it)
       /// </summary>
-      protected override void myHandleIncrementalDelete()   
-      {
+      protected override void myHandleIncrementalDelete() {
          myFireSwitch = false;
          RowDictionaryTable delGroups, addGroups;
          //form groups for deleted and added rows
-         myInitIncrementalChangeData (out delGroups, out addGroups);
-         ArrayList handledAddGroups = new ArrayList ();
+         myInitIncrementalChangeData(out delGroups, out addGroups);
+         ArrayList handledAddGroups = new ArrayList();
          string groupRepString;
 
-         foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
             // BlueEventLog.logAssert (delGroup.Count > 0);
-            groupRepString = myGetGroupString (delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
-            handledAddGroups.Add (addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
-            IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
-            myFireSwitch |= myMergeRow (newRow, groupRepString);           
+            groupRepString = myGetGroupString(delGroup);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
+            handledAddGroups.Add(addGroup);
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
+            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            myFireSwitch |= myMergeRow(newRow, groupRepString);
          }
 
-         foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
             // BlueEventLog.logAssert (addGroup.Count > 0);
-            if (!handledAddGroups.Contains (addGroup))
-            {
+            if (!handledAddGroups.Contains(addGroup)) {
                //this is the case when there are rows added to a group and none
                //have been removed, hence it 'escapes' from being dealt with in 
                //the loop above
 
-               groupRepString = myGetGroupString (addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
+               groupRepString = myGetGroupString(addGroup);
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
                // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, 
-                  new BlueTable (aggGroup.schema)));
-               myFireSwitch |= myMergeRow (newRow, groupRepString);
+               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
+                  new BlueTable(aggGroup.schema)));
+               myFireSwitch |= myMergeRow(newRow, groupRepString);
             }
          }
       }
 
-      private bool myMergeRow (IBlueRow newRow, string groupRepString)
-      {
-         IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
+      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
+         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
          //BlueEventLog.logAssert (newRow != null || existingRow != null);
          if (existingRow == null && newRow == null)
             return false;
-         else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
+         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
             return false;
-         else if (existingRow == null)
-         {
+         else if (existingRow == null) {
             //row added for the first time
-            ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-            ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
             return true;
-         }
-         else
-         {
-            ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
-            ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
-            if (newRow != null)
-            {
-               ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-               ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+         } else {
+            ((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
+            ((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
+            if (newRow != null) {
+               ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+               ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
             }
             return true;
          }
@@ -154,77 +136,68 @@ namespace BCG
       /// </summary>
       /// <param name="delGroups"></param>
       /// <param name="addGroups"></param>
-      private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
-      {
+      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
          delGroups =
-            new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
          addGroups =
-            new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
          foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
-            myBookKeepingTable.removeRow (deletedRow);
+            myBookKeepingTable.removeRow(deletedRow);
          foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
-            myBookKeepingTable.addRow (addedRow);
-         myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
+            myBookKeepingTable.addRow(addedRow);
+         myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
       }
 
-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
          if (myBookKeepingTable.Count == 0)
             return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
-         myBookKeepingTable.clear ();
-         myGroupByEvaluator.clear ();
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
+         myBookKeepingTable.clear();
+         myGroupByEvaluator.clear();
       }
 
-      private IBlueRow myEvaluate (DataPacket dp)
-      {
-         IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
-         return evalRow == null ? null : 
+      private IBlueRow myEvaluate(DataPacket dp) {
+         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
+         return evalRow == null ? null :
             myIdGuard ?
-            BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
+            BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
       }
 
-      private string myGetGroupString (IBlueRowCollection group)
-      {
-         return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
+      private string myGetGroupString(IBlueRowCollection group) {
+         return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
       }
 
-      protected override  void myOnLastSubscriber () 
-      {
+      protected override void myOnLastSubscriber() {
          //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream 
          //gets resubscribed, we start the process of building data again, and treat the callback as the first 
          //callback.
-         myInit ();
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myInit();
+         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override void myInit ()
-      {
-         base.myInit ();
-         if (myBookKeepingTable != null)
-         {
+      protected override void myInit() {
+         base.myInit();
+         if (myBookKeepingTable != null) {
             //to avoid calling init before the first callback
-            myBookKeepingTable.clear ();
-            myGroupByEvaluator.clear ();
+            myBookKeepingTable.clear();
+            myGroupByEvaluator.clear();
          }
       }
 
-      public static string getRepString (DataPacket dp, string[] colNames)
-      {
-         return BlueDataUtility.rowToString (getRepRow (dp), colNames);
+      public static string getRepString(DataPacket dp, string[] colNames) {
+         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow (DataPacket dp)
-      {
+      public static IBlueRow getRepRow(DataPacket dp) {
          if (dp.newRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.newRows, 0);
+            return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.addedRows, 0);
+            return BlueDataUtility.extractRow(dp.addedRows, 0);
          else if (dp.deletedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.deletedRows, 0);
+            return BlueDataUtility.extractRow(dp.deletedRows, 0);
          // BlueEventLog.logAssert (false, "Illegal Data Packet");
          return null;
       }
@@ -237,22 +210,21 @@ namespace BCG
       /// <param name="newValue"></param>
       /// <param name="isAdd"></param>
       /// <returns></returns>
-      public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
-      {
-         if (lastValue == null || lastValue.Equals (DBNull.Value))
+      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
+         if (lastValue == null || lastValue.Equals(DBNull.Value))
             return newValue;
          // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
-            // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
-            // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
+         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
+         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
          Object retVal = null;
-         if (newValue.GetType().Equals (typeof(int)))
+         if (newValue.GetType().Equals(typeof(int)))
             retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
-         else if (newValue.GetType().Equals (typeof(Double)))
+         else if (newValue.GetType().Equals(typeof(Double)))
             retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
-         else if (lastValue.GetType().Equals (typeof(Decimal)))
+         else if (lastValue.GetType().Equals(typeof(Decimal)))
             retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
-         else{}
-            // BlueEventLog.logAssert (false, "Illegal type");
+         else { }
+         // BlueEventLog.logAssert (false, "Illegal type");
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -22,8 +22,8 @@ namespace BCG {
             IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
             if (maxRow != null) {
                 bts.Log.assert(currMaxRow != null);
-                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
-                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
                     maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
                     if (maxRow == null)
                         myOutputTable.removeRow(hashString);
@@ -34,7 +34,7 @@ namespace BCG {
             maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
             // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
             // otherwise if maxRow > currMaxRow then its replaced
-            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
                 myOutputTable.addRow(maxRow, hashString);
             return myProject(myOutputTable.getRow(hashString));
         }