all exectuion work

The previous regex was overly ambitious (and needlessly complex as a
result). The new regex is simpler, and doesn't impose bounds on month
and day ints, aside from requiring 1-2 digits.

Bugs (fixed):
  * The DataStreamExpressionInterpretter fails to parse certain date
    constants, such as "8/30/2024"

.devcontainer/Dockerfile

@@ -0,0 +1,125 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Note installations that take a while (more than 10s)
+
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
+
+# Python: ~30s
+RUN apt-get install -y \
+   python3-pip \
+   python3-venv \
+   && python3 -m pip install --upgrade pip
+
+# GCC
+ARG gcc_major=13
+RUN \
+   add-apt-repository ppa:ubuntu-toolchain-r/test \
+   && apt-get update \
+   && apt-get install -y g++-$gcc_major
+
+# Clang
+ARG clang_major=17
+RUN \
+   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+   | tee -a "/etc/apt/sources.list.d/llvm.list" \
+   && apt-get update \
+   && apt-get install -y \
+   clang-$clang_major \
+   clang-format-$clang_major \
+   clangd-$clang_major
+
+# CMake
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
+
+# Dotnet
+RUN apt-get update && apt-get install -y dotnet7
+
+# Node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+   | tee /etc/apt/sources.list.d/nodesource.list \
+   && apt-get update \
+   && apt-get install -y nodejs
+
+# Emumba (deps)
+RUN apt-get update && apt-get -y install rapidjson-dev
+# Emumba
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
+
+# Pcap
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+
+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
+# Set up default executables (update-alternatives)
+RUN \
+   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+
+# Add compiler symlinks for ccache 
+RUN \
+   ln -s /usr/bin/ccache /usr/local/bin/clang; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+   ln -s /usr/bin/ccache /usr/local/bin/g++
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+   groupadd -g "$docker_uid" "$docker_user" \
+   && useradd \
+   -m \
+   -u "$docker_uid" \
+   -g "$docker_uid" \
+   -G sudo \
+   --password "$(openssl passwd -1 "$docker_pass")" \
+   "$docker_user"

.devcontainer/devcontainer.json

@@ -0,0 +1,50 @@
+{
+  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003",
+    "-p",
+    "12345:12345"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+  ],
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
+      "extensions": [
+        "eamodio.gitlens",
+        "llvm-vs-code-extensions.vscode-clangd",
+        "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools"
+      ]
+    }
+  }
+}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.gitattributes

@@ -0,0 +1,3 @@
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf

.gitignore

@@ -8,7 +8,9 @@ cmake-build-debug/
 .idea/
 .dir-locals.el
 *.pyc
-**.vscode
+**.vscode/*
+!.vscode/launch.json
+!.vscode/tasks.json
 .vs
 obj/
 *.csproj.user
@@ -54,6 +56,6 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-**/.devcontainer
+.devcontainer/
 *.Generated.cs
 *.lst

.vscode/launch.json

@@ -0,0 +1,158 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": ".NET Core Attach",
+      "type": "coreclr",
+      "request": "attach"
+    },
+    {
+      "name": "DataStreamPublisher",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "DataStreamQueryAnalyzer",
+      "type": "coreclr",
+      "request": "launch",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
+        "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "PubSubServer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Parameterized)",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build Spark (Debug)",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "args": [
+          "--sparkd",
+          "${input:sparkConfigServer}"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sparkd",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
+      "program": "",
+      "linux": {
+        "program": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "build/sparkd-debug-clang/server",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    }
+  ],
+  "inputs": [
+    {
+      "id": "dataStreamPublisherConfig",
+      "type": "pickString",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "sparkConfigServer",
+      "type": "promptString",
+      "description": "Config Server URL",
+      "default": "http://localhost:8000/config",
+    },
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,828 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Build Api Server",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Api Server (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Api Server (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--preset",
+          "${input:sparkdCmakePreset}",
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "${input:sparkdCompiler}"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug | Gcc)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "gcc"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug | Clang)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "clang"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "${input:sparkdCompiler}"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release | Gcc)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "gcc"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release | Clang)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "clang"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Edit DataStream Config",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "editor",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/conf/spark.conf"
+        ],
+      },
+      "windows": {
+        "command": "vim",
+        "args": [
+          "${env:APPDATA}/bts/DataStream/conf/spark.conf"
+        ],
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "Edit Spark Config",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "vim",
+        "args": [
+          "${env:APPDATA}/bts/conf/spark.conf"
+        ],
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "dedicated",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Spark",
+      "dependsOn": "Build Spark",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Release)",
+      "dependsOn": "Build Spark (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark Config Query",
+      "dependsOn": "Build Spark",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "Spark.Console",
+      },
+      "windows": {
+        "command": "Spark.Console.exe",
+      },
+      "args": [
+        "config",
+        "--query",
+        "${input:sparkConfigQuery}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd Config Query",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "config",
+        "--query",
+        "${input:sparkdConfigQuery}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "DataStreamQueryAnalyzer.exe",
+          "'Starting DataStreamQueryAnalyzer'"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+  ],
+  "inputs": [
+    {
+      "id": "buildConfiguration",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "Debug",
+        "Release",
+      ],
+      "default": "Debug",
+    },
+    {
+      "id": "sparkdCmakePreset",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "sparkd-debug-clang",
+        "sparkd-debug-gcc",
+        "sparkd-release-clang",
+        "sparkd-release-gcc",
+      ],
+      "default": "sparkd-debug-clang",
+    },
+    {
+      "id": "sparkdCompiler",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "clang",
+        "gcc",
+      ],
+      "default": "sparkd-debug-clang",
+    },
+    {
+      "id": "dataStreamPublisherConfig",
+      "type": "pickString",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "sparkConfigQuery",
+      "type": "promptString",
+      "description": "Config Type(s)"
+    },
+    {
+      "id": "sparkdConfigQuery",
+      "type": "promptString",
+      "description": "Config Type(s)"
+    }
+  ]
+}

CMakeLists.txt

@@ -14,12 +14,17 @@ set(
   CACHE
   STRING
   "External dependencies root folder (should contain folders: lib, bin, include")
+
+# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
+# can run simultaneously without fighting over the same shm files (e.g., sparkd,
+# api server, etc.).
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
 set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
 set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
-set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
+set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
 if (WIN32)
   set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
   set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -31,13 +36,13 @@ endif()
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})
 
-if(BUILD_API_SERVER)
-  add_compile_definitions(SPARK_ENABLE_API_SERVER)
+add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
-
+add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
-  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
-  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
   # subdirs
   add_subdirectory(src/lib)
 else()
@@ -47,13 +52,7 @@ else()
   set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
   set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
 
-  if(BUILD_API_SERVER)
-    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
-    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
-  elseif(BUILD_SPARKD)
-    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
-    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
-
+  if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
     add_compile_definitions(SPARK_ENABLE_CCAPI)
@@ -133,6 +132,8 @@ else()
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
       COMMAND
         ln ARGS -sf
         ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0

CMakePresets.json

@@ -15,11 +15,13 @@
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build",
+      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -30,7 +32,12 @@
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
       }
     },
 
@@ -46,6 +53,8 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "ON"
@@ -62,7 +71,8 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
       }
     },
 
@@ -95,98 +105,72 @@
     {
       "name": "sparkd-debug-clang",
       "inherits": "sparkd-debug",
-      "displayName": "Debug (clang)",
+      "displayName": "Sparkd (Debug | Clang)",
       "description": "Debug build compiled w/ clang",
-      "binaryDir": "${sourceDir}/build/clang/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
       }
     },
 
     {
       "name": "sparkd-debug-gcc",
       "inherits": "sparkd-debug",
-      "displayName": "Debug (gcc)",
+      "displayName": "Sparkd (Debug | Gcc)",
       "description": "Debug build compiled w/ gcc",
-      "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
       }
     },
 
     {
       "name": "sparkd-release-clang",
       "inherits": "sparkd-release",
-      "displayName": "Release (clang)",
+      "displayName": "Sparkd (Release | Clang)",
       "description": "Release build compiled w/ clang",
-      "binaryDir": "${sourceDir}/build/clang/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
       }
     },
 
     {
       "name": "sparkd-release-gcc",
       "inherits": "sparkd-release",
-      "displayName": "Release (gcc)",
+      "displayName": "Sparkd (Release | Gcc)",
       "description": "Release build compiled w/ gcc",
-      "binaryDir": "${sourceDir}/build/gcc/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
-      }
-    },
-
-    {
-      "name": "spark-api-server-base",
-      "hidden": true,
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
       }
     },
 
     {
       "name": "spark-api-server-windows",
       "inherits": "spark",
-      "displayName": "Spark Api Server",
+      "displayName": "Spark Api Server (Windows)",
       "cacheVariables": {
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        }
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
       }
     },
 
     {
       "name": "spark-api-server-linux",
       "inherits": "sparkd",
-      "displayName": "Spark Api Server",
+      "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
         },
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        },
         "BUILD_SPARKD": {
           "type": "BOOL",
           "value": "OFF"
@@ -197,7 +181,11 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
       }
     },
 
@@ -220,13 +208,62 @@
     },
 
     {
-      "name": "data-stream-suite",
+      "name": "data-stream-suite-windows",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
       "inherits": "spark",
-      "displayName": "Data Stream Suite",
+      "displayName": "Data Stream Suite (Windows)",
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
+    },
+
+    {
+      "name": "data-stream-suite-linux",
+      "displayName": "Data Stream Suite (Linux)",
+      "description": "Data Stream utils ported over from Edge",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "inherits": "sparkd",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
       }
     }
   ],
@@ -244,7 +281,7 @@
 
     {
       "name": "spark-debug",
-      "displayName": "Debug",
+      "displayName": "Spark (Debug)",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Debug"
@@ -252,7 +289,7 @@
 
     {
       "name": "spark-release",
-      "displayName": "Release",
+      "displayName": "Spark (Release)",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Release"
@@ -280,28 +317,28 @@
     {
       "name": "sparkd-debug-clang",
       "configurePreset": "sparkd-debug-clang",
-      "displayName": "Debug (clang)",
+      "displayName": "Sparkd (Debug | Clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-debug-gcc",
       "configurePreset": "sparkd-debug-gcc",
-      "displayName": "Debug (gcc)",
+      "displayName": "Sparkd (Debug | Gcc)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-clang",
       "configurePreset": "sparkd-release-clang",
-      "displayName": "Release (clang)",
+      "displayName": "Sparkd (Release | Clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-gcc",
       "configurePreset": "sparkd-release-gcc",
-      "displayName": "Release (gcc)",
+      "displayName": "Sparkd (Release | Gcc)",
       "inherits": "sparkd"
     },
 
@@ -318,14 +355,14 @@
 
     {
       "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Windows | Debug)",
       "inherits": "spark-api-server-windows",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Windows | Release)",
       "inherits": "spark-api-server-windows",
       "configuration": "Release"
     },
@@ -344,14 +381,14 @@
     {
       "name": "spark-api-server-linux-debug",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-linux-release",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Release)",
+      "displayName": "Spark Api Server (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -371,16 +408,30 @@
     },
 
     {
-      "name": "data-stream-suite-debug",
-      "displayName": "DataStreamSuite (Debug)",
-      "configurePreset": "data-stream-suite",
+      "name": "data-stream-suite-windows-debug",
+      "displayName": "Data Stream Suite (Windows | Debug)",
+      "configurePreset": "data-stream-suite-windows",
       "configuration": "Debug"
     },
 
     {
-      "name": "data-stream-suite-release",
-      "displayName": "DataStreamSuite (Release)",
-      "configurePreset": "data-stream-suite",
+      "name": "data-stream-suite-windows-release",
+      "displayName": "Data Stream Suite (Windows | Release)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "data-stream-suite-linux-debug",
+      "displayName": "Data Stream Suite (Linux | Debug)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-linux-release",
+      "displayName": "Data Stream Suite (Linux | Release)",
+      "configurePreset": "data-stream-suite-linux",
       "configuration": "Release"
     }
   ]

playground-1.mongodb.js

@@ -1,45 +0,0 @@
-/* global use, db */
-// MongoDB Playground
-// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
-// Make sure you are connected to enable completions and to be able to run a playground.
-// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
-// The result of the last command run in a playground is shown on the results panel.
-// By default the first 20 documents will be returned with a cursor.
-// Use 'console.log()' to print to the debug output.
-// For more documentation on playgrounds please refer to
-// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
-
-// Select the database to use.
-use('mongodbVSCodePlaygroundDB');
-
-// Insert a few documents into the sales collection.
-db.getCollection('sales').insertMany([
-    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
-    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
-]);
-
-// Run a find command to view items sold on April 4th, 2014.
-const salesOnApril4th = db.getCollection('sales').find({
-    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
-}).count();
-
-// Print a message to the output window.
-console.log(`${salesOnApril4th} sales occurred in 2014.`);
-
-// Here we run an aggregation and open a cursor to the results.
-// Use '.toArray()' to exhaust the cursor to return the whole result set.
-// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
-db.getCollection('sales').aggregate([
-    // Find all of the sales that occurred in 2014.
-    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
-    // Group the total sales for each product.
-    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
-]);
-
-

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BaseUtilities/Crono.cs

@@ -1,41 +1,22 @@
-using mbts.spark;
+using System.Diagnostics;
 
-using System;
-using System.Runtime;
+namespace BCG.Utilities.General;
 
-namespace BCG.Utilities.General
-{
-	/// <summary>
-	/// Summary description for Crono.
-	/// </summary>
-	public class Crono
-	{
-      public Crono()
-      {
-         myTimer = new HighPerformanceTimer();
-      }
+/// <summary>
+/// Summary description for Crono.
+/// </summary>
+public class Crono {
+   readonly Stopwatch myTimer = new();
 
-      public void start() 
-      {myTimer.start();}
+   public void start() =>
+      myTimer.Start();
 
-      public void stop()
-      {myTimer.stop();}
+   public void stop() =>
+      myTimer.Stop();
 
-      public TimeSpan time
-      {
-         get 
-         {
-            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
-         }
-      }
-
-      public double miliseconds
-      {
-         get
-         {return myTimer.totalMilliseconds;}
-      }
-
-      private HighPerformanceTimer myTimer;
-	}
+   public TimeSpan time =>
+      myTimer.Elapsed;
 
+   public double miliseconds =>
+      myTimer.Elapsed.TotalMilliseconds;
 }

src/DataStreamSuite/BlueRegistry/BlueRegistry.cs

@@ -1,192 +1,192 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Reflection;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public enum RegistryRoot { HKCU, HKLM }
-
-   [Serializable]
-   public class BlueRegistry
-   {
-      public const string BlueSubKey = @"Software\Blue Capital Group\";
-      public static readonly string ApplicationPrefix;
-      protected string myFullPath;
-
-      public BlueRegistry(string subPath)
-      {
-         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
-      }
-
-      public BlueRegistry(BlueRegistry parent, string subPath)
-      {
-         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
-      }
-
-      static BlueRegistry()
-      {
-         string app = "Applications";
-
-         Assembly ass = Assembly.GetEntryAssembly();
-         if (ass != null)
-         {
-            app = ass.GetName().Name;
-
-            object[] attrs = ass.GetCustomAttributes(
-               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
-
-            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
-            {
-               app = ((AssemblyTitleAttribute)attrs[0]).Title;
-            }
-         }
-
-         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
-      }
-
-      public object getValue(string name)
-      {
-         return getValue(RegistryRoot.HKCU, name);
-      }
-
-      public object getValue(string name, object defaultValue)
-      {
-         return getValue(RegistryRoot.HKCU, name, defaultValue);
-      }
-
-      public void setValue(string name, object value)
-      {
-         setValue(RegistryRoot.HKCU, name, value);
-      }
-
-      public void deleteValue(string name)
-      {
-         deleteValue(RegistryRoot.HKCU, name);
-      }
-
-      private RegistryKey myGetRegistryKey(RegistryRoot root)
-      {
-         if (root == RegistryRoot.HKCU)
-            return Registry.CurrentUser.CreateSubKey(myFullPath);
-         else
-            return Registry.LocalMachine.CreateSubKey(myFullPath);
-      }
-
-      #region Generic Methods
-      internal virtual void setValue(RegistryRoot root, string name, object value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.SetValue(name, value);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name);
-         }
-      }
-
-      internal virtual void deleteValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.DeleteValue(name);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name, defaultValue);
-         }
-      }
-      #endregion
-
-      #region Double
-      internal virtual void setDouble(RegistryRoot root, string name, double value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            DoubleRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual double getDouble(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return DoubleRegistryHandler.getValue(rk, name);
-         }
-      }
-      #endregion
-
-      #region Enum
-      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            EnumRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return EnumRegistryHandler.getValue(rk, name, enumType);
-         }
-      }
-      #endregion
-
-      #region Array
-      internal virtual void setArray(RegistryRoot root, string name, Array array)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            ArrayRegistryHandler.setValue(rk, name, array);
-         }
-      }
-
-      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return ArrayRegistryHandler.getValue(rk, name, elementType);
-         }
-      }
-      #endregion
-
-      #region SortedList
-      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.setValue(rk, name, slist);
-         }
-      }
-
-      internal virtual SortedList getSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return SortedListRegistryHandler.getValue(rk, name);
-         }
-      }
-
-      internal virtual void deleteSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.deleteValue(rk, name);
-         }
-      }
-      #endregion
-   }
-}
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      public virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      public virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,10 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-    <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>
 
 </Project>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnExpression">
+    <Scanner Include="ColumnExpressionTokens.lex" />
+    <Parser Include="ColumnExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -0,0 +1,62 @@
+namespace Bts.ColumnExpressions;
+
+public class ColumnExpression {
+   public ColumnExpression(string name, Type? type = null, object? value = null) {
+      Name = name;
+      Type = type
+         ?? value?.GetType()
+         ?? throw new ArgumentException(
+            "Type must be specified explicitly when value is null", nameof(type));
+      Value = value
+         ?? DBNull.Value;
+   }
+
+   public string Name { get; }
+
+   public Type Type { get; }
+
+   public object Value { get; }
+
+   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
+      // Try to convert value to 'type' to verify they are compatible
+      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
+      // account for implicit conversions (e.g., int -> long), just inheritance.
+      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
+         throw new ArgumentException(
+            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
+      }
+      return new(name, type ?? typeof(T), value);
+   }
+
+   internal static Type ParseType(string typeId) {
+      return Candidates().ExceptNull().FirstOrDefault() ??
+         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
+
+      IEnumerable<Type?> Candidates() {
+         if (typeId.Length == 0) {
+            yield break;
+         }
+         yield return Type.GetType(typeId);
+         yield return Type.GetType($"System.{typeId}");
+         if (typeId.Length > 1) {
+            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
+         }
+         yield return typeId.ToLower() switch {
+            "float64" or "double" or "real64" or "number" => typeof(double),
+            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
+            "int16" or "short" or "integer16" or "signed16" => typeof(short),
+            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
+            "int64" or "long" or "integer64" or "signed64" => typeof(long),
+            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
+            "uint32" or "uint" or "unsigned" => typeof(uint),
+            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
+            "char" => typeof(char),
+            "byte" or "uint8" => typeof(byte),
+            "bool" or "boolean" => typeof(bool),
+            "string" or "text" => typeof(string),
+            "date" or "datetime" => typeof(DateTime),
+            _ => null
+         };
+      }
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -0,0 +1,49 @@
+%namespace Bts.ColumnExpressions
+%partial
+%visibility internal
+%tokentype ColumnExpressionToken
+%parsertype ColumnExpressionParser
+
+%union {
+   public int Integer;
+   public double Real;
+   public bool Boolean;
+   public string String;
+   public string Identifier;
+   public string TypeId;
+   public DateTime DateTime;
+   public Tuple<string, Type> ColumnDefinition;
+   public ColumnExpression ColumnExpression;
+   public ImmutableList<ColumnExpression> ColumnExpressions;
+}
+
+%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
+%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
+
+%nonassoc EQUALS
+
+%type <ColumnDefinition> ColumnDefinition
+%type <ColumnExpression> ColumnExpression
+%type <ColumnExpressions> ColumnExpressions
+
+%%
+
+ColumnExpressions :
+   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
+   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
+   ;
+
+ColumnExpression :
+   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
+   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
+   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
+   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
+   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
+   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
+   ;
+
+ColumnDefinition :
+   IDENTIFIER { $$ = new($1.Identifier, null); }
+   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
+   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
+   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnExpressions;
+
+using System.IO;
+using System.Text;
+
+public class ColumnExpressionInterpreter {
+   public static IList<ColumnExpression> Evaluate(Stream stream) {
+      var parser = new ColumnExpressionParser();
+      var res = parser.Parse(stream);
+      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
+         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
+   }
+
+   public static IList<ColumnExpression> Evaluate(string str) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
+      return Evaluate(ms);
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnExpressions;
+
+using System.Text;
+
+partial class ColumnExpressionParser {
+   public ColumnExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -0,0 +1,9 @@
+namespace Bts.ColumnExpressions;
+
+partial class ColumnExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -0,0 +1,85 @@
+%namespace Bts.ColumnExpressions
+%scannertype ColumnExpressionScanner
+%visibility internal
+%tokentype ColumnExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+BooleanConstant          true|false
+StringConstant           '([^\'\\]*|\\.)*'
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+IntegerConstant          -?[0-9]+
+NullConstant             'null'
+RealConstant             -?[0-9]+\.[0-9]+
+
+/* Relational Operators */
+Equals                   =
+OfType                   /
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+NamespacePrefix          {Identifier}\.
+TypeId                   {NamespacePrefix}*{Identifier}
+
+/* Utility Tokens */
+OpenParenthesis          \(
+CloseParenthesis         \)
+Comma                    ,
+QuoteOpen                \[
+QuoteClose               \]
+
+/* Comments */
+LineComment \/\/[^\n\r]*(\n|\r|\r\n)
+
+%%
+
+{Equals} {
+   return (int)ColumnExpressionToken.EQUALS;
+}
+
+{OfType} {
+   return (int)ColumnExpressionToken.OF_TYPE;
+}
+
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
+   return (int)ColumnExpressionToken.STRING_CONSTANT;
+}
+
+{BooleanConstant} {
+   yylval.Boolean = yytext.ToLower() switch {
+      "true" => true,
+      "false" => false,
+      _ => throw new Exception(
+         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
+   };
+   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
+}
+
+{DateTimeConstant} {
+   yylval.DateTime = DateTime.Parse(yytext);
+   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
+}
+
+{IntegerConstant} {
+   yylval.Integer = int.Parse(yytext);
+   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
+}
+
+{RealConstant} {
+   yylval.Real = double.Parse(yytext);
+   return (int)ColumnExpressionToken.REAL_CONSTANT;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnExpressionToken.IDENTIFIER;
+}
+
+{TypeId} {
+   yylval.TypeId = yytext;   
+   return (int)ColumnExpressionToken.TYPE_ID;
+}
+
+%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnSortExpression">
+    <Scanner Include="ColumnSortExpressionTokens.lex" />
+    <Parser Include="ColumnSortExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -0,0 +1,11 @@
+namespace Bts.ColumnSortExpressions;
+
+public class ColumnSortExpression {
+   public ColumnSortExpression(string name, bool isAscending = true) {
+      Name = name;
+      IsAscending = isAscending;
+   }
+
+   public string Name { get; }
+   public bool IsAscending { get; }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -0,0 +1,29 @@
+%namespace Bts.ColumnSortExpressions
+%partial
+%visibility internal
+%tokentype ColumnSortExpressionToken
+%parsertype ColumnSortExpressionParser
+
+%union {
+   public string Identifier;
+   public ColumnSortExpression Column;
+   public ImmutableList<ColumnSortExpression> Columns;
+}
+
+%token ASCENDING, DESCENDING, IDENTIFIER
+
+%type <Column> Column
+%type <Columns> Columns
+
+%%
+
+Columns :
+   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
+   | Columns Column { $$ = $1.Add($2); }
+   ;
+
+Column :
+   IDENTIFIER { $$ = new($1.Identifier, true); }
+   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
+   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
+   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnSortExpressions;
+
+using System.IO;
+using System.Text;
+
+public class ColumnSortExpressionInterpreter {
+   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
+      var parser = new ColumnSortExpressionParser();
+      var res = parser.Parse(stream);
+      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
+         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
+   }
+
+   public static IList<ColumnSortExpression> Evaluate(string str) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
+      return Evaluate(ms);
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnSortExpressions;
+
+using System.Text;
+
+partial class ColumnSortExpressionParser {
+   public ColumnSortExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnSortExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -0,0 +1,9 @@
+namespace Bts.ColumnSortExpressions;
+
+partial class ColumnSortExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -0,0 +1,30 @@
+%namespace Bts.ColumnSortExpressions
+%scannertype ColumnSortExpressionScanner
+%visibility internal
+%tokentype ColumnSortExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+Ascending                +
+Descending               -
+
+%%
+
+{Ascending} {
+   return (int)ColumnSortExpressionToken.ASCENDING;
+}
+
+{Descending} {
+   return (int)ColumnSortExpressionToken.DESCENDING;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnSortExpressionToken.IDENTIFIER;
+}
+
+%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -9,6 +9,8 @@
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -0,0 +1,26 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Reactive.Linq;
+
+public static class DataStreamExtensions {
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
+      source as ViewStream ?? new ObservableViewStream(source);
+
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
+         source.onChange += OnChange;
+         return Disposable.Create(() => source.onChange -= OnChange);
+
+         void OnChange(object? newValue) =>
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
+      }
+   }
+}

src/DataStreamSuite/DataStream/Misc/PandasScriptStream.cs

@@ -0,0 +1,29 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+using System.IO;
+
+public class PandasScriptStream : ViewStream {
+   public PandasScriptStream(ViewStream vs, string file)
+      : this(vs, file, null) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
+      : this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
+      mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
+
+      string readFile() {
+         try {
+            return File.ReadAllText(file);
+         } catch (Exception ex) {
+            Log.Gen.error($"PandasScriptStream error reading file: {ex}");
+            throw;
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -0,0 +1,130 @@
+namespace Bts.DataStream;
+
+using BCG;
+using BCG.Extensions;
+using BCG.Utilities.General;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+public class PandasStream : ViewStream, IWorker {
+   readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
+   readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
+   readonly string pythonExe;
+   readonly ViewStreamDatabase vsd;
+   readonly CSVSaverStream inputCsvStream;
+   readonly Lazy<string> pythonCode;
+   IDictionary<int, IBlueRow>? lastRowMap = null;
+   int inputVersion = 0;
+   int outputVersion = 0;
+   DateTime lastFireTime = DateTime.MinValue;
+   TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
+   IDisposable? dispose;
+
+   public PandasStream(ViewStream vs, string dataFrameExpression)
+      : this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
+
+   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
+      pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
+      this.pythonExe = pythonExe;
+      this.vsd = vsd;
+      // This should fire an update when it writes to the output file
+      inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
+      pythonCode = new(getPythonCode);
+
+      string getPythonCode() {
+         var imports =
+@"import numpy as np
+import pandas as pd
+import sys
+";
+         var bd = new StringBuilder(imports);
+         bd.AppendLine($"df=pd.read_csv('{inputFile}')");
+         bd.AppendLine(dataFrameExpression);
+         bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
+         return bd.ToString();
+      }
+   }
+
+   protected override void myOnFirstSubscriber() {
+      vsd.addWorker(this);
+      inputCsvStream.onChange += onInputChanged;
+      this.dispose = Disposable.Create(dispose);
+
+      void onInputChanged(object newValue) =>
+         inputVersion++;
+
+      void dispose() {
+         inputCsvStream.onChange -= onInputChanged;
+         vsd.removeWorker(this);
+         deleteFile(inputFile);
+         deleteFile(outputFile);
+      }
+
+      void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
+         if (!File.Exists(file)) {
+            return;
+         }
+         try {
+            File.Delete(file);
+         } catch (Exception ex) {
+            Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref dispose);
+
+   static void executePythonCode(string pythonCode, string? pythonExe = null) {
+      pythonExe ??= "python";
+      var start = new ProcessStartInfo {
+         FileName = pythonExe,
+         Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
+         UseShellExecute = false,
+         RedirectStandardOutput = true
+      };
+      using var process = Process.Start(start)
+         ?? throw new InvalidOperationException("Failed to launch python");
+      process.WaitForExit();
+   }
+
+   void IWorker.doWork() {
+      if (outputVersion == inputVersion) {
+         return;
+      }
+      var now = DateTime.Now;
+      var elapsedSinceLastFire = now - lastFireTime;
+      if (elapsedSinceLastFire < fireThrottle) {
+         return;
+      }
+      outputVersion = inputVersion;
+      executePythonCode(pythonCode.Value, pythonExe);
+      var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
+      var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
+      var comp = BlueRowEqualityComparer.Instance;
+      var deletedRows = new BlueTable(schema);
+      var addedRows = new BlueTable(schema);
+      if (lastRowMap is not null) {
+         deletedRows.SetItems(lastRowMap.Where(oldRow =>
+            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
+         addedRows.SetItems(rowMap.Where(newRow =>
+            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
+      }
+      lastRowMap = rowMap;
+      lastFireTime = DateTime.Now;
+      myFireEvent(new DataPacket(rows, addedRows, deletedRows));
+   }
+}

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -0,0 +1,79 @@
+using BCG.Utilities.General;
+
+using System;
+
+namespace BCG {
+    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
+        class State {
+            public int n_sends = 0;
+            public int n_recvs = 0;
+        }
+        private readonly string _ip;
+        private readonly int _port;
+        private readonly ViewStreamDatabase _vsd;
+        State _state = new State();
+        private readonly int _msg_size;
+
+        private IBlueTableSchema _schema;
+        private TimeSpan _send_period;
+        private ITCPConnection _connection = null;
+        private byte[] _buff;
+
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
+        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
+            _ip = ip;
+            _port = port;
+            _msg_size = msg_size;
+            _vsd = vsd;
+            _schema = BlueDataUtility.form_schema("TCPClient",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("n_sends", TypeCode.UInt32),
+            Tuple.Create("n_recvs", TypeCode.UInt32));
+            _send_period = TimeSpan.FromSeconds(send_period_seconds);
+            _buff = new byte[msg_size];
+            for (int i = 0; i < _buff.Length; i++) {
+                _buff[i] = (byte)i;
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, _send_period);
+            var server_rci = new RemoteConnectionInfo(_ip, _port);
+            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            if (_connection != null) {
+                _connection.send(_buff);
+                _state.n_sends++;
+            }
+            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
+            BlueTable rows = new BlueTable(r.schema);
+            rows.addRow(r);
+            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        public void onConnection(ITCPConnection connection) {
+            _connection = connection;
+            _connection.register(this);
+        }
+        public bool isDisposed { get; }
+
+        public void onData(RemoteMessage msg) {
+            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
+            _state.n_recvs++;
+        }
+
+        public void Dispose() {
+
+        }
+
+
+    }
+}

src/DataStreamSuite/DataStream/Misc/test.py

@@ -0,0 +1,3 @@
+file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
+with open(file_path, "w") as f:
+    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -0,0 +1,37 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
+   IDisposable? subscription;
+
+   public ObservableViewStream(IObservable<DataPacket> source) =>
+      this.source = source;
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      if (subscription is not null) {
+         subscription.Dispose();
+         subscription = null;
+      }
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(subscription);
+      subscription = source.Subscribe(myFireEvent);
+   }
+
+   protected override void myOnLastSubscriber() {
+      Log.notnull(subscription);
+      subscription.Dispose();
+      subscription = null;
+   }
+
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
+      source.Subscribe(observer);
+}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,13 +1,7 @@
-using BCG;
-
 using System;
 using System.Collections.Generic;
 
-using mbts.config;
-using bts.utils;
-using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
-using System.Collections.Generic;
 using BCG.Utilities.General;
 using bts;
 
@@ -30,8 +24,10 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream build_stream(string stream_expression) {
-         try {
+      static ViewStream? build_stream(string stream_expression)
+      {
+         try
+         {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
             var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
@@ -55,10 +51,13 @@ namespace BCG {
             _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
          });
       }
-      // add a RemoteQuery sub, additionally and subscribe to it
-      private void _on_str(object obj) {
-         DataPacket dp = obj as DataPacket;
-         foreach (IBlueRow r in dp.newRows) {
+      private void _on_str(object obj)
+      {
+         if (obj is not DataPacket dp) {
+            return;
+         }
+         foreach (IBlueRow r in dp.newRows)
+         {
             string name = r.get<string>("name");
             bool persist = r.get<bool>("persist");
             string stream_exp = r.get<string>("stream_exp");
@@ -88,4 +87,4 @@ namespace BCG {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,10 +1,6 @@
 using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using BCG.Utilities.General;
-namespace BCG {
 
+namespace BCG {
    public class AddColumnsStream : ViewStream {
       public static Tuple<string, Type, object> parse_exp(string exp) {
          string[] toks = exp.Split(':');
@@ -35,4 +31,4 @@ namespace BCG {
          return rv;
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -0,0 +1,12 @@
+namespace bts {
+    using BCG;
+
+    using System;
+    using System.Collections;
+
+    public class AddConstantColumnRow {
+        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,63 +1,52 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Evaluator.AddColumnsExpression;
-
-// namespace BCG {
-//     /// <summary>
-//     /// Provides methods to add columns to the schema
-//     /// </summary>
-//     public class ColumnAdder : IStreamMapper {
-//         protected IDictionary myNewColumns;
-//         protected IBlueTableSchema mySchema;
-
-//         static syntax myParser =
-//            new syntax();
-
-//         /// <summary>
-//         /// string represting the columns to be added to the output table. 
-//         /// </summary>
-//         /// <param name="colAddExpression"></param>
-//         public ColumnAdder(string columnDef) {
-//             AddColumnsExpression ace;
-
-//             if (DataStreamLoadFileConfig.instance.useStaticParser) {
-//                 lock (myParser) {
-//                     ace =
-//                     (AddColumnsExpression)myParser.Parse(columnDef);
-//                 }
-//             } else {
-//                 syntax parser = new syntax();
-//                 ace =
-//                 (AddColumnsExpression)parser.Parse(columnDef);
-//             }
-
-//             if (ace == null)
-//                 throw new Exception("Invalid columnn expression: " + columnDef);
-//             myNewColumns = ace.columns;
-//         }
-
-//         public virtual IBlueRow map(IBlueRow row) {
-//             if (mySchema == null)
-//                 myConstructSchema(row.schema);
-//             return new AddColumnsRow(mySchema, row);
-//         }
-
-//         public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-//             myConstructSchema(inputSchema);
-//             return mySchema;
-//         }
-
-//         private void myConstructSchema(IBlueTableSchema schema) {
-//             ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
-//             int ciIndex = 0;
-//             foreach (ColumnInfo ci in schema.columnInfo)
-//                 cis[ciIndex++] = ci;
-//             foreach (DictionaryEntry de in myNewColumns)
-//                 cis[ciIndex] = new ColumnInfo((string)de.Key, ciIndex++,
-//                    ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
-//             mySchema = new BlueTableSchema(cis, schema.tableName);
-//         }
-
-//     }
-// }
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+using Bts.ColumnExpressions;
+
+namespace BCG;
+
+/// <summary>
+/// Provides methods to add columns to the schema
+/// </summary>
+/// <param name="columnsExpression">
+/// String represting the columns to add to the output table
+/// </param>
+public class ColumnAdder : IStreamMapper {
+   // Store tuples of (column, index) to preserve order
+   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
+   protected IBlueTableSchema? mySchema;
+
+   public ColumnAdder(string columnsExpression) =>
+      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
+         throw new Exception("Invalid columnn expression: " + columnsExpression))
+         .Select((c, i) => (c, i))
+         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
+
+   [MemberNotNull(nameof(mySchema))]
+   public virtual IBlueRow map(IBlueRow row) {
+      // Only rebuild schema if necessary (i.e., if it's null)
+      mySchema ??= buildSchema(row.schema, myNewColumns);
+      return new AddColumnsRow(mySchema, row);
+   }
+
+   [MemberNotNull(nameof(mySchema))]
+   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
+      // Always rebuild schema (even if it's already non-null)
+      mySchema = buildSchema(inputSchema, myNewColumns);
+
+   [MemberNotNull(nameof(mySchema))]
+   static BlueTableSchema buildSchema(
+      IBlueTableSchema schema,
+      IDictionary<string, (ColumnExpression column, int index)> newColumns
+   ) {
+      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
+      schema.columnInfo.CopyTo(cis, 0);
+      foreach (var (col, index) in newColumns.Values) {
+         var schemaIdx = schema.columnInfo.Length + index;
+         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
+      }
+      return new BlueTableSchema(cis, schema.tableName);
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs

@@ -1,23 +1,24 @@
+using bts;
+
 using System;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation ("distinct")]
-	public class DistinctRowStream : ViewStream
-	{
-		public DistinctRowStream (ViewStream stream, params string[] cols)
-		{
-         ArrayList groupByCols = new ArrayList ();
-         foreach (string col in cols)
-            foreach (string c in col.Split (' '))
-               groupByCols.Add (c);
-         mySetRealSource (new GroupByStream (stream, 
-            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
+	[DataStreamUserRespresentation("distinct")]
+	public class DistinctRowStream : ViewStream {
+		public DistinctRowStream(ViewStream stream, params string[] cols) {
+			var g_by_eval = new DistinctRowEvaluator(cols);
+			ViewStream r = new GroupByStream(stream, g_by_eval);
+			var proj_exp = String.Empty;
+			foreach (var c in cols)
+				proj_exp += $"{c} ";
+			proj_exp = proj_exp.Trim();
+			r = new NewProjectStream(r, proj_exp);
+			mySetRealSource(r);
 		}
 	}
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,6 +1,7 @@
 using BCG;
 
 namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;
 
-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -16,6 +17,11 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
+   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
+   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
+   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
+   // file is created. The file names are token_epoch_datetime
+   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;
@@ -89,10 +95,10 @@ namespace BCG {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
          _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -100,8 +106,8 @@ namespace BCG {
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -5,7 +5,8 @@ using System.IO;
 using BCG;
 
 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
    [DataStreamUserRespresentation("saver")]
    public class SaverStream : ViewStream {
       private ViewStream _stream;
@@ -31,7 +32,6 @@ namespace bts {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
          byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
          StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -9,15 +9,29 @@
 // using System.Reflection.Metadata;
 
 
-// namespace BCG {
-//    public class SnapshotStream : ViewStream {
-
+namespace BCG {
+   public class SnapshotStream : ViewStream {
+      public enum SpecificSnapshot {
+         FirstOfDay,
+         LastOfDay
+      };
 //       private BlueTable _current_table;
 //       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
 //       public DateTime _snapshot_time;
 //       public string _snapshot_file;
 //       public IEnumerable<PlaybackPacket> _packets;
-
+//       public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+//          DateTime dt;
+//          var parsed = DateTime.TryParse(snapshot_date, out dt);
+//          dt = dt.Date;
+//          bts.Log.assert(parsed);
+//          SpecificSnapshot specific_snapshot_enum;
+//          parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+//          bts.Log.assert(parsed);
+//          string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+//          ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+//          mySetRealSource(rs);
+//       }
 
 //       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
 //          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
@@ -111,5 +125,5 @@
 
 //          }
 //       }
-//    }
-// }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;
 
 
 namespace BCG {
@@ -37,6 +38,20 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
+         DateTime max_dt = DateTime.MinValue;
+         string rv = String.Empty;
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            var dt = parse_datetime_from_filename(fn);
+            if (dt > max_dt) {
+               max_dt = dt;
+               rv = fn;
+            }
+         }
+         return rv;
+      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,16 +1,10 @@
-using BCG;
-using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using System.Text.RegularExpressions;
-
 using Bts.DataStream;
 
-using mbts;
-using mbts.spark;
 using mbts.config;
 
+using System;
+using System.Text.RegularExpressions;
+
 namespace BCG.StreamWorld.Addition
 {
    [DataStreamUserRespresentation (false)]
@@ -151,7 +145,6 @@ namespace BCG
       private string myFilterString;
       private BCG.StreamWorld.Addition.FilterStream myFilterStream;
       private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -226,7 +219,6 @@ namespace BCG
 
       private void myHandleCallbacks(object newValue)
       {
-         _timer.start();
          DataPacket retVal;
          DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
@@ -255,7 +247,6 @@ namespace BCG
                if (myFilter.filter(r))
                   myLastFiredRows.addRow(r);
          }
-         _timer.stop();
          myFireEvent(retVal);
       }
 

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,47 +1,42 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
+using System;
+using System.Collections;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Computes the max for a given column
-// 	/// </summary>
-// 	public class MaxGroupByEvaluator : GroupByEvaluator
-// 	{
+using BCG.Utilities;
 
-//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the max for a given column
+    /// </summary>
+    public class MaxGroupByEvaluator : GroupByEvaluator {
 
-//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
-//             myOutputTable;
-//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
-//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
-//          if (maxRow != null)
-//          {
-//             BlueEventLog.logAssert (currMaxRow != null);
-//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
-//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
-//             {
-//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
-//                if (maxRow == null)
-//                   myOutputTable.removeRow (hashString);
-//                else
-//                   myOutputTable.addRow (maxRow, hashString);
-//             }
-//          }
-//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
-//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-//          // otherwise if maxRow > currMaxRow then its replaced
-//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
-//             myOutputTable.addRow (maxRow, hashString);
-//          return myProject (myOutputTable.getRow (hashString));
-//       }
-// 	}
-// }
+        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
+
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
+            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
+            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
+            if (maxRow != null) {
+                bts.Log.assert(currMaxRow != null);
+                bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
+                    if (maxRow == null)
+                        myOutputTable.removeRow(hashString);
+                    else
+                        myOutputTable.addRow(maxRow, hashString);
+                }
+            }
+            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
+            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+            // otherwise if maxRow > currMaxRow then its replaced
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
+                myOutputTable.addRow(maxRow, hashString);
+            return myProject(myOutputTable.getRow(hashString));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,20 +1,15 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-// 	public class MaxStream : ViewStream
-// 	{
-//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
-//       {
-//       }
-// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
-// 		{
-//          mySetRealSource (new GroupByStream (stream, 
-//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
-// 		}
+namespace BCG {
+    public class MaxStream : ViewStream {
+        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
+        }
+        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
+            mySetRealSource(new GroupByStream(stream,
+                                                new MaxGroupByEvaluator(outputCol, inputCols)));
+        }
 
-//       public MaxStream (ViewStream stream, string outputCol)
-//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-// 	}
-// }
+        public MaxStream(ViewStream stream, string outputCol)
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,40 +1,36 @@
-// using System;
-// using BCG.Utilities;
+using System;
 
-// namespace BCG
-// {
-//    public class MinGroupByEvaluator : MaxGroupByEvaluator
-//    {
-//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
-//          : base (outputCol, groupByCols)  {}
+using BCG.Utilities;
 
-//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
-//          base (colMapEpression, outputCol, groupByCols)  {}
+namespace BCG {
+    public class MinGroupByEvaluator : MaxGroupByEvaluator {
+        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
-//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
-//          if (minRow != null)
-//          {
-//             // BlueEventLog.logAssert (currMinRow != null);
-//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
-//             {
-//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
-//                if (minRow == null)
-//                   myOutputTable.removeRow (stringRep);
-//                else
-//                   myOutputTable.addRow (minRow, stringRep);
-//             }
-//          }
-//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
-//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
-//             myOutputTable.addRow (minRow, stringRep);
-//          return myOutputTable.getRow (stringRep);
-//       }
+        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
+           base(colMapEpression, outputCol, groupByCols) { }
 
-//    }
-// }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
+            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
+            if (minRow != null) {
+                // BlueEventLog.logAssert (currMinRow != null);
+                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+                if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
+                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
+                    if (minRow == null)
+                        myOutputTable.removeRow(stringRep);
+                    else
+                        myOutputTable.addRow(minRow, stringRep);
+                }
+            }
+            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
+                myOutputTable.addRow(minRow, stringRep);
+            return myOutputTable.getRow(stringRep);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,17 +1,13 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class MinStream : ViewStream
-//    {
-//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
-//       {
-//          mySetRealSource (new GroupByStream (stream, 
-//                            new MinGroupByEvaluator (outputCol, inputCol)));
-//       }
+namespace BCG {
+    public class MinStream : ViewStream {
+        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
+            mySetRealSource(new GroupByStream(stream,
+                              new MinGroupByEvaluator(outputCol, inputCol)));
+        }
 
-//       public MinStream (ViewStream stream, string outputCol)
-//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-//    }
-// }
+        public MinStream(ViewStream stream, string outputCol)
+                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,10 +1,8 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using mbts;
-using mbts.spark;
+using System.Diagnostics;
+
+using bts;
 
 namespace BCG
 {
@@ -14,7 +12,7 @@ namespace BCG
       private INonIncGroupByEvaluator _groupby_func;
       private BlueRowHasher _row_hasher;
       private string[] _groupby_cols;
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();
+      private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
       public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
@@ -33,21 +31,27 @@ namespace BCG
       {
          DataPacket dp = nv as DataPacket;
          ///grouping together
-         _timer.start();
+         _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.stop();
-         // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         _timer.Stop();
+         var groupElapsedMs = _timer.ElapsedMilliseconds;
+         if (groupElapsedMs >= 50) {
+            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
+         }
+         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
-         _timer.start();
+         _timer.Restart();
          IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.stop();
+         _timer.Stop();
+         var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;         
-         if (_timer.lastIntervalTime > 50)
+         if (applyElapsedMs >= 50)
          {
             _n_expensive_fires++;
-            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
+            Log.DataFlow.info(
+               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }         
          myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
-      }      
+      }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,94 +1,82 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Computes the sum for a given column
-//    /// </summary>
-//    public class SumGroupByEvaluator : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the sum for a given column
+    /// </summary>
+    public class SumGroupByEvaluator : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-         
-//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+
+            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
 
-//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
-//       {
-//          myOutputCols = output_columns;
-//          myGroupByCols = groupby_columns;
-//       }
+        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
+            myOutputCols = output_columns;
+            myGroupByCols = groupby_columns;
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-//    public class SumGroupByEvaluator<T> : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-//          : base(outputCol, groupByCols)
-//       { }
+    public class SumGroupByEvaluator<T> : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-//          : base(colMapEpression, outputCol, groupByCols) { }
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate(DataPacket dp)
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
-//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-// }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,107 +1,89 @@
-// using System;
-// using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
-// namespace BCG
-// {
-//    public class NullMapper : IStreamMapper
-//    {
-//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+namespace BCG {
+    public class NullMapper : IStreamMapper {
+        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-//       public NullMapper(string[] valid_cols)
-//       {
-//          foreach (string c in valid_cols)
-//             _valid_cols[c] = true;
-//          _valid_cols["id"] = true;
-//       }
+        public NullMapper(string[] valid_cols) {
+            foreach (string c in valid_cols)
+                _valid_cols[c] = true;
+            _valid_cols["id"] = true;
+        }
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-//       public IBlueRow map(IBlueRow row)
-//       {
-//          MutableBlueRow out_row = new MutableBlueRow(row);
-//          foreach (string c in row.schema.columnNames)
-//             if (!_valid_cols.ContainsKey(c))
-//                out_row.setValue(c, DBNull.Value);
-//          return out_row;
-//       }
-//    }
+        public IBlueRow map(IBlueRow row) {
+            MutableBlueRow out_row = new MutableBlueRow(row);
+            foreach (string c in row.schema.columnNames)
+                if (!_valid_cols.ContainsKey(c))
+                    out_row.setValue(c, DBNull.Value);
+            return out_row;
+        }
+    }
 
-//    public class SumStream : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//       {
-//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//       }
+    public class SumStream : ViewStream {
+        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns)
-//       {
-//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-//          rs = new ProjectStream(rs, "!fake_col");
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns) {
+            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+            rs = new ProjectStream(rs, "!fake_col");
+            mySetRealSource(rs);
+        }
 
-//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
-//       {
-//          string[] output_columns = space_seperated_output_columns.Split(' ');
-//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
-//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
-//       }
+        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
+            string[] output_columns = space_seperated_output_columns.Split(' ');
+            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
+            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
-//       {
-//          ViewStream rs = null;
-//          groupby_columns = groupby_columns ?? new string[] {};
-//          if (groupby_columns.Length == 0)
-//          {
-//             rs = new AddColumnsStream(stream, "fake_col=0");
-//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
-//             rs = new ProjectStream(rs, "!fake_col");
-//          }
-//          else
-//          {
-//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-//          }
-//          if (force_null_into_other_columns)
-//          {
-//             List<string> valid_cols = new List<string>(output_columns);
-//             valid_cols.AddRange(groupby_columns);
-//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-//          }
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
+            ViewStream rs = null;
+            groupby_columns = groupby_columns ?? new string[] { };
+            if (groupby_columns.Length == 0) {
+                // rs = new AddColumnsStream(stream, "fake_col=0");
+                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
+                // rs = new ProjectStream(rs, "!fake_col");
+            } else {
+                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+            }
+            if (force_null_into_other_columns) {
+                List<string> valid_cols = new List<string>(output_columns);
+                valid_cols.AddRange(groupby_columns);
+                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+            }
+            mySetRealSource(rs);
+        }
 
-//    public SumStream (ViewStream stream, string outputCol)
-//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
-//    {
-//    }
-// }
+        public SumStream(ViewStream stream, string outputCol)
+        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
+        }
+    }
 
-// public class IntSumStream : ViewStream
-// {
-//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//    {
-//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//       mySetRealSource(new GroupByStream(stream,
-//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
-//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//    }
+    public class IntSumStream : ViewStream {
+        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-// }
+    }
 
 
-//    public class SumStream<T> : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
-//       {
-//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
-//       }
-//    }
-// }
+    public class SumStream<T> : ViewStream {
+        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
+            string[] inputCols = spaceSeparatedInputCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -0,0 +1,79 @@
+using BCG;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+using ZstdSharp.Unsafe;
+
+namespace bts {
+    public class RowDiffStream : ViewStream {
+        private ViewStream _vs;
+        private readonly string _order_col, _col_1, _col_2;
+        private readonly int _diff;
+        private IBlueTableSchema _out_schema;
+        private readonly string _diff_col_name;
+        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
+            _vs = vs;
+            _order_col = order_col;
+            _col_1 = col_1;
+            _col_2 = col_2;
+            _diff = diff;
+            _diff_col_name = diff_col_name;
+        }
+
+        private string _diff_col_name_perc {
+            get {
+                return $"{_diff_col_name}_percentage";
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _handle_vs;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _handle_vs;
+        }
+
+        private void _handle_vs(object dp_obj) {
+            var dp = dp_obj as DataPacket;
+            IBlueRowCollection nrows = dp.newRows;
+            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
+            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
+            List<IBlueRow> row_list = new List<IBlueRow>();
+            foreach (IBlueRow r in nrows)
+                row_list.Add(r);
+            //OrderBy sorts in ascedning order
+            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
+            var sorted_row_array = sorted_rows.ToArray();
+            // reverse it to make it descending    
+            sorted_rows.Reverse();
+            //sort rows so we can do the diff
+            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
+            int i = 0;
+            foreach (IBlueRow r in sorted_rows) {
+                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
+                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
+                // we assume x - y, and name variables like that
+                double x = r.get<double>(_col_1);
+                int y_row_idx = i + _diff;
+                double diff = System.Double.NaN;
+                double perc = System.Double.NaN;
+                if (y_row_idx < sorted_rows.Count()) {
+                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
+                    diff = x - y;
+                    perc = 100.0 * diff / y;
+                }
+                nr.setValue(_diff_col_name, diff);
+                nr.setValue(_diff_col_name_perc, perc);
+                rv.addRow(nr);
+                i++;
+            }
+
+            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs

@@ -27,7 +27,8 @@ namespace BCG
             string prependString = isLeft ? "left" : "right";
             return String.Concat (prependString, colName).GetHashCode ();
          }
-         public override bool Equals (object obj)  {return GetHashCode () == obj.GetHashCode ();}
+         public override bool Equals (object? obj)  =>
+            GetHashCode() == obj?.GetHashCode();
       }
 
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs

@@ -10,9 +10,9 @@ namespace BCG
 {
    public class JoinStream : ViewStream
    {
-      protected InternalJoinStream myIjs;
-      protected ProjectStream myPs;
-      protected ColumnMap myLeftCm, myRightCm;
+      protected InternalJoinStream? myIjs;
+      protected ProjectStream? myPs;
+      protected ColumnMap? myLeftCm, myRightCm;
       protected bool myIsProject = true;
       private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
       

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,10 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using bts;
+
 using mbts;
-using mbts.spark;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
 
 namespace BCG
 {
@@ -40,8 +40,7 @@ namespace BCG
 
       public void _on_stream_cb(object dp_obj)
       {
-         HighPerformanceTimer timer = new HighPerformanceTimer();
-         timer.start();
+         var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
          if (_schema == null)
          {
@@ -67,13 +66,12 @@ namespace BCG
                   _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
             }
          }
-         timer.stop();
+         timer.Stop();
          _n_total_fires++;
-         if (timer.lastIntervalTime > 50)
-         {
+         if (timer.ElapsedMilliseconds >= 50) {
             _n_expensive_fires++;
-            string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
-            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
+            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
          }
          if (on_new_data != null)
             on_new_data();

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -1,10 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using bts;
+
 using mbts;
-using mbts.spark;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
 
 namespace BCG
 {
@@ -114,8 +114,7 @@ namespace BCG
          if (_is_ready())
          {
             _construct_schema();
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
+            var timer = Stopwatch.StartNew();
             IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
             List<object[]> output_rows = new List<object[]>();
 
@@ -138,13 +137,12 @@ namespace BCG
                }
             }
             IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.stop();
+            timer.Stop();
             _n_total_fires++;
-            if (timer.lastIntervalTime >= 250)
-            {
+            if (timer.ElapsedMilliseconds >= 250) {
                _n_expensive_fires++;
-               string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
-               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
+               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
             }
             myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
@@ -154,15 +152,12 @@ namespace BCG
       {
          if (_output_mapper != null)
          {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
             IBlueRowCollection output_rows;
-            timer.start();
+            var timer = Stopwatch.StartNew();
             output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.stop();
-            if (timer.lastIntervalTime >= 50)
-            {
-               // string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
-               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
+            timer.Stop();
+            if (timer.ElapsedMilliseconds >= 50) {
+               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
             }
             return output_rows;
          }

src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs

@@ -1,11 +1,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using BCG.Utilities;
 using mbts;
-using mbts.spark;
 using Spark.Core.Collections;
 
 namespace BCG
@@ -226,7 +222,8 @@ namespace BCG
 
       #region IDictionary Members
 
-      public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
+      public void Add(object key, object? value) =>
+         throw new NotSupportedException("IBlueRow is readonly");
 
       public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
 
@@ -256,10 +253,9 @@ namespace BCG
          }
       }
 
-      public object this[object key]
-      {
-         get { return key is string ? this[key.ToString()] : this[(int)key]; }
-         set { throw new NotSupportedException("IBlueRow is readonly"); }
+      public object? this[object key] {
+         get => key is string str ? this[str] : this[(int)key];
+         set => throw new NotSupportedException("IBlueRow is readonly");
       }
 
       #endregion
@@ -294,7 +290,6 @@ namespace BCG
       protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
                                           myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
       private bool myManageUnmappedLeftRows = false;
-      protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
       protected string myLeftProj = String.Empty, myRightProj = String.Empty;
       protected IBlueTableSchema myLeftSchema, myRightSchema;
       protected JoinProjectHelper myJoinProjectHelper;
@@ -361,7 +356,6 @@ namespace BCG
 
       private void myHandleStreamChanges(DataPacket dp, bool isLeft)
       {
-         myTimer.start();
          myInitSchema(dp.schema, isLeft);
          BlueTable dels = null, adds = null;
          if (myState != null)
@@ -371,7 +365,6 @@ namespace BCG
          }
          myDeleteRows(dp.deletedRows, isLeft, myState, dels);
          myAddRows(dp.addedRows, isLeft, myState, adds);
-         myTimer.stop();
          if (myState != null)
          {
             myFireEvent(new DataPacket(myState, adds, dels));

src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs

@@ -91,7 +91,8 @@ namespace BCG
          }
       }
 
-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add (object key, object? value) =>
+         throw new NotSupportedException ("The IBlueRow is readonly");
 
       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
 

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -1,13 +1,9 @@
-using System;
-using System.Data;
-using System.Collections;
-
 using Bts.DataStream;
 
-using mbts;
-using mbts.spark;
 using mbts.config;
 
+using System;
+
 namespace BCG {
 
 
@@ -17,9 +13,9 @@ namespace BCG {
    /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
    /// is preserved across the rows, ie row["id"] = f(row)["id"]
    /// </summary>
-   [DataStreamUserRespresentation(false)]
-   public class MapStream : MethodDrivenStream {
-      private HighPerformanceTimer t = new HighPerformanceTimer();
+   [DataStreamUserRespresentation (false)]
+   public class MapStream : MethodDrivenStream
+   {
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -8,26 +8,33 @@ using System;
 using System.IO;
 using System.Linq;
 
-namespace bts {
-    public class PubSubTestingStream : ViewStream, IWorker {
-        public class PayloadMessage {
+namespace bts
+{
+    public class PubSubTestingStream : ViewStream, IWorker
+    {
+        public class PayloadMessage
+        {
             public int msg_id = 0;
             public string guid;
             public Int64 travel_epoch_ms = 0;
             public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
+            {
                 msg_id = msg_id_args;
                 travel_epoch_ms = send_epoch_args;
                 payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++) {
+                for (int i = 0; i < payload_size; i++)
+                {
                     payload[i] = payload_byte;
                 }
             }
             private PayloadMessage() { }
             public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
+            { }
 
-            public byte[] serialize() {
+            public byte[] serialize()
+            {
                 var str = new MemoryStream();
                 BinaryWriter bw = new BinaryWriter(str);
                 bw.Write(msg_id);
@@ -40,7 +47,8 @@ namespace bts {
                 return bytes;
             }
 
-            public static PayloadMessage deserialize(byte[] ser_bytes) {
+            public static PayloadMessage deserialize(byte[] ser_bytes)
+            {
                 var str = new MemoryStream(ser_bytes);
                 BinaryReader br = new BinaryReader(str);
                 int msg_id = br.ReadInt32();
@@ -48,7 +56,8 @@ namespace bts {
                 Int64 epoch = br.ReadInt64();
                 int len = br.ReadInt32();
                 var payload = br.ReadBytes(len);
-                return new PayloadMessage() {
+                return new PayloadMessage()
+                {
                     msg_id = msg_id,
                     guid = guid,
                     travel_epoch_ms = epoch,
@@ -73,7 +82,8 @@ namespace bts {
         private readonly int _payload_sz;
         private const byte _payload_byte = (byte)'A';
         private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
+        {
             _vsd = vsd;
             _payload_sz = payload_size;
             var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -96,43 +106,53 @@ namespace bts {
         public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
         : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
 
-        protected override void myOnFirstSubscriber() {
+        protected override void myOnFirstSubscriber()
+        {
             _vsd.addWorker(this);
             System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
             th.IsBackground = true;
             th.Start();
         }
 
-        protected override void myOnLastSubscriber() {
+        protected override void myOnLastSubscriber()
+        {
             _vsd.removeWorker(this);
             _continue_sub_thread = false;
         }
 
-        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
+        void _log_updates(IEnumerable<PubSubServerMessage> updates)
+        {
             var idx = 0;
-            foreach (var msg in updates) {
+            foreach (var msg in updates)
+            {
                 bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
                 idx++;
             }
         }
 
-        private void _subscribe_thread_func() {
+        private void _subscribe_thread_func()
+        {
             // subscription is done in another thread
             SubscriberCore sub_core = new SubscriberCore();
             sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread) {
+            while (_continue_sub_thread)
+            {
                 // TODO recv_id
                 var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
                 if (updates == null || updates.Count() == 0)
                     continue;
                 _log_updates(updates);
                 _n_updates += updates.Count();
-                if (updates.Count() > 0) {
+                if (updates.Count() > 0)
+                {
                     var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0) {
+                    if (server_msg.data != null && server_msg.data.Length > 0)
+                    {
                         _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
                         _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    } else {
+                    }
+                    else
+                    {
                         // why are we getting empty or null data ? (server is sending data regardless?)
                     }
                     _fire_dirty = true;
@@ -140,28 +160,36 @@ namespace bts {
             }
         }
 
-        private void _publish_next_packet() {
-            if (_last_sent_msg == null) {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
+        private void _publish_next_packet()
+        {
+            if (_last_sent_msg == null)
+            {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
+                {
                     guid = Guid.NewGuid().ToString()
                 };
-            } else {
+            }
+            else
+            {
                 _last_sent_msg.msg_id++;
                 _last_sent_msg.guid = Guid.NewGuid().ToString();
                 _last_sent_msg.travel_epoch_ms = epoch_now();
             }
             var ser_bytes = _last_sent_msg.serialize();
             _n_pubs++;
-            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
+            _pub_core.publish("Blob", _tag_name, ser_bytes);
+            _fire_dirty = true;
         }
 
-        private void _fire_dp() {
+        private void _fire_dp()
+        {
             object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
             object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
             object echo_time = DBNull.Value;
             object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
             object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null) {
+            if (_last_sent_msg != null && _last_recvd_msg != null)
+            {
                 echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
             }
             var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -173,21 +201,25 @@ namespace bts {
             myFireEvent(dp);
         }
 
-        public void doWork() {
+        public void doWork()
+        {
             if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs) {
+                if (_n_pubs <= _max_pubs)
+                {
                     _publish_next_packet();
                     _last_publish_time = DateTime.Now;
                 }
             // publish 
             // TODO: send id
-            if (_fire_dirty) {
+            if (_fire_dirty)
+            {
                 _fire_dirty = false;
                 _fire_dp();
             }
         }
 
-        public static long epoch_now() {
+        public static long epoch_now()
+        {
             return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
         }
 

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -0,0 +1,223 @@
+using BCG;
+using BCG.Utilities.General;
+using System.Collections.Generic;
+
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using bts.api;
+
+namespace bts
+{
+   [DataStreamUserRespresentation("pss_echo")]
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
+         public readonly string msg_id;
+         public readonly Int64 timestamp = 0;
+         public readonly byte[] payload = null;
+
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
+            var rv = new byte[payload_size];
+            for (int i = 0; i < payload_size; i++)
+            {
+               rv[i] = payload_byte;
+            }
+            return rv;
+         }
+
+         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
+         // {
+         //    guid = guid_;
+         //    timestamp = timestamp_;
+         //    payload = new byte[payload_size];
+         //    for (int i = 0; i < payload_size; i++)
+         //    {
+         //       payload[i] = payload_byte;
+         //    }
+         // }
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
+            msg_id = guid_;
+            timestamp = timestamp_;
+            payload = payload_;
+         }
+
+         private PayloadMessage() { }
+         public byte[] serialize()
+         {
+            var str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            bw.Write(msg_id);
+            bw.Write(timestamp);
+            bw.Write(payload.Length);
+            bw.Write(payload);
+            str.Position = 0;
+            var bytes = str.ToArray();
+            return bytes;
+         }
+
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
+            var str = new MemoryStream(ser_bytes);
+            BinaryReader br = new BinaryReader(str);
+            string guid = br.ReadString();
+            Int64 timestamp_ = br.ReadInt64();
+            int len = br.ReadInt32();
+            var payload = br.ReadBytes(len);
+            return new PayloadMessage(guid, timestamp_, payload);
+         }
+      }
+      //public void
+      private readonly int _msg_size;
+      private readonly TimeSpan _publish_period;
+      private DateTime _last_publish_time = DateTime.MinValue;
+      private PublisherCore _pub_core;
+      private ISubscriberCore _sub_core;
+      private BlueTable _table;
+      private int _n_pubs = 0;
+      private readonly int _max_pubs;
+      private int _n_updates = 0;
+      private ViewStreamDatabase _vsd;
+      private PayloadMessage _last_sent_msg, _last_recvd_msg;
+      private const byte _payload_byte = (byte)'A';
+      private readonly string _tag_name;
+      private readonly int _payload_sz;
+      private readonly IPubSubRouter _router;
+      private enum IOState
+      {
+         SEND = 0,
+         RECV = 1
+      };
+      IOState _state = IOState.SEND;
+
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
+      {
+         var route = new PubSubRoute(server_url, ".*");
+         _router = new PubSubRouter(route);
+         _vsd = vsd;
+         _payload_sz = payload_size;
+         var schema = BlueDataUtility.form_schema("PubSubTest",
+         Tuple.Create("id", TypeCode.Int32),
+         Tuple.Create("sent_id", TypeCode.String),
+         Tuple.Create("sent_time", TypeCode.Int64),
+         Tuple.Create("recv_id", TypeCode.String),
+         Tuple.Create("recv_time", TypeCode.Int64),
+         Tuple.Create("match", TypeCode.Boolean),
+         Tuple.Create("echo_time", TypeCode.Int32));
+         _table = new BlueTable(schema);
+         _publish_period = publish_period;
+         _tag_name = tag_name;
+      }
+
+      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
+      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _pub_core = new PublisherCore(_router);
+         _sub_core = new SubscriberCore(_router);
+         _sub_core.subscribe(_tag_name);
+         _vsd.addWorker(this);
+
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         _vsd.removeWorker(this);
+         _sub_core.unsubscribe(_tag_name);
+      }
+
+      private void _fire_dp_after_send()
+      {
+         // as soon as the data is published, its reflected in the stream state
+         Log.assert(_last_sent_msg != null);
+         BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
+            DBNull.Value, DBNull.Value);
+         _table.addRow(r);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+
+      private void _fire_dp_after_recv()
+      {
+         Log.assert(_last_recvd_msg != null);
+         // update the last publish row
+         int id = _table.Count - 1;
+         IBlueRow last_row = _table[id];
+         _table.deleteRow(last_row);
+         var recv_ts = epoch_now();
+         BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
+                                 String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
+         _table.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+
+      private string _get_next_id()
+      {
+         return _table.Count.ToString();
+      }
+
+      private void _publish_next_packet()
+      {
+         var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
+         _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
+         var ser_bytes = _last_sent_msg.serialize();
+         _n_pubs++;
+         _pub_core.publish("Blob", _tag_name, ser_bytes);
+         _last_publish_time = DateTime.Now;
+      }
+
+      public void doWork()
+      {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
+         {
+            foreach (var u in updates)
+            {
+               var recv_msg = PayloadMessage.deserialize(u.data);
+               if (recv_msg.msg_id == _last_sent_msg.msg_id)
+               {
+                  return recv_msg;
+               }
+            }
+            return null;
+         };
+         switch (_state)
+         {
+            case IOState.SEND:
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
+                  _publish_next_packet();
+                  _last_publish_time = DateTime.Now;
+                  _fire_dp_after_send();
+                  _state = IOState.RECV;
+                  return;
+               }
+               break;
+            case IOState.RECV:
+               var updates = _sub_core.get_updates(TimeSpan.Zero);
+               updates = updates.Where(u => u.tag == _tag_name);
+               var recv_msg = find_packet(updates);
+               if (recv_msg != null)
+               {
+                  _last_recvd_msg = recv_msg;
+                  _fire_dp_after_recv();
+                  _state = IOState.SEND;
+                  break;
+               }
+               break;
+         }
+      }
+
+      public static long epoch_now()
+      {
+         return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs

@@ -14,11 +14,14 @@ namespace bts {
       private BlueTable _table;
       private SyncQueue<string> _action_queue = new SyncQueue<string>();
 
-      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) {
+      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
+      : this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
+
+      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
          _ip = ip;
          _port = port;
          _table = new BlueTable(_schema);
-         _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms);
+         _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
       }
 
       protected override void myOnFirstSubscriber() {

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -1,17 +1,8 @@
-
-
-using Amazon.Runtime.Internal;
-
 using bts;
 
-using DnsClient.Protocol;
-
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System;
-using Microsoft.VisualBasic;
-using Spark.Core.Collections;
 using System.Reflection;
 
 namespace BCG {
@@ -37,7 +28,8 @@ namespace BCG {
 
         private IBlueTableSchema _schema;
         class TestResult {
-            public string test_name = MethodBase.GetCurrentMethod().Name;
+            public string test_name = MethodBase.GetCurrentMethod()?.Name
+               ?? throw new InvalidOperationException("Failed to get test name from method");
             public bool test_result;
         }
         private List<Func<TestResult>> _tests = new
@@ -51,11 +43,11 @@ namespace BCG {
             Tuple.Create("exception", TypeCode.String)
             );
             // register tests here
-            // _tests.Add(new Func<TestResult>(_run_sub_msg_test));
-            // _tests.Add(new Func<TestResult>(_run_pub_msg_test));
-            // _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
-            // _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
-            // _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
+            _tests.Add(new Func<TestResult>(_run_sub_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pub_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
+            _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
             // _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
             // _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
             //  _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
@@ -814,4 +806,4 @@ namespace BCG {
 
 
     }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TickingTestStream.cs

@@ -0,0 +1,31 @@
+namespace BCG {
+   using System;
+
+   public class TickingTestStream : ViewStream {
+      public TickingTestStream(int ticking_period_msecs)
+      :
+      this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
+
+      public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
+         ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
+         vs = new MathStream(vs, "x=id*123 y=x*x");
+         mySetRealSource(vs);
+         //vs.onChange += _on_str_update;
+      }
+
+      // private void _on_str_update(object dp_obj) {
+      //    DataPacket dp = dp_obj as DataPacket;
+      //    IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
+      //    BlueTable state = new BlueTable(out_schema);
+      //    foreach (IBlueRow r in dp.newRows) {
+      //       MutableBlueRow nr = new MutableBlueRow(out_schema, r);
+      //       DateTime time = r.get_datetime("time");
+      //       var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+      //       nr.setValue("time_txt", time_txt);
+      //       state.addRow(nr);
+      //    }
+      //    var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
+      //    myFireEvent(dp_out);
+      // }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs

@@ -0,0 +1,194 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using Bts.DataStreamCore;
+
+using mbts;
+
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Reactive.Subjects;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
+   readonly string collectionName;
+   readonly string databaseName;
+   readonly string url;
+   readonly object syncRoot = new();
+   readonly CancellationTokenSource cts = new();
+   readonly Subject<DataPacket> subject = new();
+
+   bool hasStarted = false;
+   IBlueRowFactory<BsonDocument>? rowFactory;
+   MongoClient? clientBacking;
+   IMongoDatabase? databaseBacking;
+   IMongoCollection<BsonDocument>? collectionBacking = null;
+   BlueTableImmutable? table = null;
+
+   public MongoCollectionStream(string collectionName)
+      : this(
+         collectionName,
+         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName, string databaseName)
+      : this(
+         collectionName,
+         databaseName,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName, string databaseName, string url)
+      : this(collectionName, databaseName, url, null) { }
+
+   internal MongoCollectionStream(
+      string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
+      this.collectionName = collectionName;
+      this.databaseName = databaseName;
+      this.url = url;
+      this.rowFactory = rowFactory;
+      mySetRealSource(new ObservableViewStream(this));
+   }
+
+   MongoClient client => clientBacking ??= new(url);
+   IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
+   IMongoCollection<BsonDocument>? collection {
+      get => Volatile.Read(ref collectionBacking);
+      set => Volatile.Write(ref collectionBacking, value);
+   }
+
+   public IDisposable Subscribe(IObserver<DataPacket> observer) {
+      lock (syncRoot) {
+         if (hasStarted) {
+            if (table is not null) {
+               // The first update has already been fired. Seed observer w/ current state.
+               observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
+            }
+            return subject.Subscribe(observer);
+         }
+         hasStarted = true;
+      }
+      var subscription = subject.Subscribe(observer);
+      // Retrieve collection snapshot to bootstrap sequence
+      initTable().ignore();
+      // Watch changes
+      Task.Run(watchChanges);
+      return Disposable.Create(Dispose);
+
+      void Dispose() {
+         Log.Gen.info("Disposing MongoDbChangeObservable subscription");
+         subscription.Dispose();
+         cts.Cancel();
+         cts.Dispose();
+      }
+   }
+
+   protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
+      MongoSchemaAdaptor.InferSchema(docs);
+
+   async Task watchChanges() {
+      Log.Gen.info(
+         $"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}");
+      while (collection is null) {
+         await Task.Delay(100);
+      }
+      Log.notnull(collection);
+      using var cursor = collection.Watch(new ChangeStreamOptions {
+         FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
+      });
+      try {
+         while (await cursor.MoveNextAsync(cts.Token)) {
+            /// Ignore while waiting for init.
+            lock (syncRoot) {
+               /// Note: <see cref="rowFactory"/> and <see cref="table"/> are initialized together
+               /// in the <see cref="initTable"/> fn since both depend on the schema, which is
+               /// inferred from the first incoming BsonDocument(s).
+               if (table is null || rowFactory is null) {
+                  continue;
+               }
+            }
+
+            var bd = table.ToDataPacketBuilder();
+            foreach (var ch in cursor.Current) {
+               var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
+               var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
+               Log.Gen.info(
+                  $"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
+                  $"changes:\n\t\t{updDescJson}");
+               switch (ch.OperationType) {
+                  case ChangeStreamOperationType.Insert: {
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Update: {
+                        bd.Remove(OldRow().Id());
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Replace: {
+                        bd.Remove(OldRow().Id());
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Delete: {
+                        bd.Remove(OldRow().Id());
+                        break;
+                     }
+                  default:
+                     Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
+                     break;
+               }
+
+               IBlueRow Row() =>
+                  rowFactory.GetBlueRow(ch.FullDocument);
+
+               IBlueRow OldRow() =>
+                  rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
+            }
+            var dp = bd.ToDataPacket();
+            if (!dp.HasChanges()) {
+               Log.Gen.warn(
+                  $"MongoCollectionStream. Ignoring update because it didn't change any rows.");
+               continue;
+            }
+            lock (syncRoot) {
+               table = (BlueTableImmutable)dp.newRows;
+               subject.OnNext(dp);
+            }
+         }
+      } catch (TaskCanceledException ex) {
+         Log.Gen.warn(
+            $"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}");
+      }
+   }
+
+
+   [MemberNotNull(nameof(collection))]
+   async Task initTable() {
+      collection = database.GetCollection<BsonDocument>(collectionName);
+      var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+      var docList = await cursor.ToListAsync();
+      rowFactory ??= getRowFactory(docList);
+      var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
+      foreach (var doc in docList) {
+         var row = rowFactory.GetBlueRow(doc);
+         firstTable.AddOrUpdate(row);
+      }
+      var dp = firstTable.ToDataPacket();
+      lock (syncRoot) {
+         table = (BlueTableImmutable)dp.newRows;
+         subject.OnNext(dp);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,130 +1,133 @@
-using System;
-using System.Collections.Generic;
-using MongoDB.Driver;
-using MongoDB.Bson;
 using BCG;
 using BCG.Utilities.General;
-using BCG;
-using mbts.spark;
 
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Threading;
 
 namespace bts {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        // make this a sorted table
-        private BlueTable _table;
+    // [DataStreamUserRespresentation("mongodb_sql")]
+    // public class MongoDBSQLStream : ViewStream, IWorker {
+    //     private ViewStreamDatabase _vsd;
+    //     private readonly string _connection_string, _db_name, _collection_name;
+    //     private MongoClient _client;
+    //     // make this a sorted table
+    //     private BlueTable _table;
 
 
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+    //     // Access the database
+    //     private IMongoDatabase _database;
+    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+    //     // mongodb://192.168.1.80:27017
 
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+    //     }
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+    //                                 $"db_name: {db_name}");
+    //         _connection_string = connection_string;
+    //         _db_name = db_name;
+    //         _collection_name = collection_name;
+    //         _vsd = vsd;
+    //         _client = new MongoClient(_connection_string);
+    //         _database = _client.GetDatabase(_db_name);
+    //     }
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
-        }
+    //     protected override void myOnFirstSubscriber() {
+    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+    //     }
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+    //     protected override void myOnLastSubscriber() {
+    //         MasterWorker.getInstance(false).removeWorker(this);
+    //     }
 
-        public void doWork() {
-            _read_mogodb();
-        }
+    //     public void doWork() {
+    //         _read_mogodb();
+    //     }
 
-        private void _read_mogodb() {
+    //     private void _read_mogodb() {
+    //         var timer = Stopwatch.StartNew();
+    //         // Access the collection
+    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+    //         // Fetch all documents from the collection
+    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+    //         _table = _table == null ? null : new BlueTable(_table.schema);
+    //         // Iterate through the documents and print values
+    //         foreach (var document in documents.ToEnumerable()) {
+    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+    //             if (_table == null) {
+    //                 _init_table(document);
+    //             }
+    //             _add_row(document);
+    //         }
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+    //         timer.Restart();
+    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+    //             return;
+    //         myFireEvent(dp);
+    //     }
 
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables: {timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+    //     private void _init_table(BsonDocument doc_row) {
+    //         bts.Log.assert(_table == null);
+    //         List<ColumnInfo> cis = new List<ColumnInfo>();
+    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+    //         foreach (var n in doc_row.Names) {
+    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+    //         }
+    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+    //         _table = new BlueTable(schema);
+    //     }
 
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
-
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
+    //     private void _add_row(BsonDocument doc) {
+    //         var key_val = doc["_id"];
+    //         var key = key_val.ToString();
+    //         int id;
+    //         if (!_id_map.TryGetValue(key, out id)) {
+    //             id = _id_map.Count;
+    //             _id_map.Add(key, id);
+    //         }
+    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
+    //         nr.setValue("id", id);
+    //         foreach (var n in doc.Names) {
+    //             var val = doc[n].ToString();
+    //             nr.setValue(n, val);
+    //         }
+    //         _table.addRow(nr);
+    //     }
+    // }
 
     [DataStreamUserRespresentation("mongodb")]
     public class MongoDBStream : ViewStream, IWorker {
         private ViewStreamDatabase _vsd;
         private readonly string _connection_string, _db_name, _collection_name;
+        private readonly TimeSpan _poll_period;
         private MongoClient _client;
         private BlueTable _table;
+        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
+        private BlockingQueue _work_queue = new BlockingQueue();
 
         // Access the database
         private IMongoDatabase _database;
         private IDictionary<string, int> _id_map = new Dictionary<string, int>();
         // mongodb://192.168.1.80:27017
 
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
+        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
 
         }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
             bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                     $"db_name: {db_name}");
             _connection_string = connection_string;
@@ -132,24 +135,34 @@ namespace bts {
             _collection_name = collection_name;
             _vsd = vsd;
             _client = new MongoClient(_connection_string);
+            _poll_period = poll_period;
             _database = _client.GetDatabase(_db_name);
         }
 
         protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+            _vsd.addWorker(this);
+            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
         }
 
         protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
+            _vsd.removeWorker(this);
+            //MasterWorker.getInstance(false).removeWorker(this);
         }
 
+        private DateTime _last_poll_time = DateTime.MinValue;
         public void doWork() {
-            _read_mogodb();
+            var now = DateTime.Now;
+            if (now - _last_poll_time > _poll_period) {
+                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
+                _last_poll_time = DateTime.Now;
+            }
+            var dp = _table_queue.dequeue(0);
+            if (dp != null)
+                myFireEvent(dp);
         }
 
         private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
+            var timer = Stopwatch.StartNew();
             // Access the collection
             IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
@@ -164,15 +177,16 @@ namespace bts {
                 }
                 _add_doc(document);
             }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
+            timer.Stop();
+            Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
+            timer.Restart();
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            timer.Stop();
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
             if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                 return;
-            myFireEvent(dp);
+            _table_queue.enqueue(dp);
+            //myFireEvent(dp);
         }
 
         private void _init_table(BsonDocument doc_row) {
@@ -211,46 +225,46 @@ namespace bts {
         }
     }
 
-    [DataStreamUserRespresentation("mongodb_flat")]
-    public class MongoDBFlatStream : ViewStream {
-        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
-        :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+    // [DataStreamUserRespresentation("mongodb_flat")]
+    // public class MongoDBFlatStream : ViewStream {
+    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+    //     :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
 
-        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
-            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
-            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
-            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
-            mySetRealSource(rs);
-        }
-    }
+    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+    //         mySetRealSource(rs);
+    //     }
+    // }
 
 
 
-    [DataStreamUserRespresentation("newsreplay")]
-    public class NewsReplayStream : ViewStream {
-        public NewsReplayStream(string connection_string, string db_name, string collection_name,
-                                int buffer_sz, string fire_period) :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+    // [DataStreamUserRespresentation("newsreplay")]
+    // public class NewsReplayStream : ViewStream {
+    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, string fire_period) :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
 
-        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
-                                int buffer_sz, TimeSpan fire_period) {
-            ViewStream r;
+    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, TimeSpan fire_period) {
+    //         ViewStream r;
 
-            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            r = new KeyValueToRelationalStream(r,
-                                                "doc_id",
-                                                "key",
-                                                "value");
-            r = new DateStandardizationStream(r,
-                                               "created_at",
-                                               "ISO8601");
-            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         r = new KeyValueToRelationalStream(r,
+    //                                             "doc_id",
+    //                                             "key",
+    //                                             "value");
+    //         r = new DateStandardizationStream(r,
+    //                                            "created_at",
+    //                                            "ISO8601");
+    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
 
-            mySetRealSource(r);
-        }
-    }
+    //         mySetRealSource(r);
+    //     }
+    // }
 
 
-}
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs

@@ -0,0 +1,63 @@
+namespace Bts.DataStream;
+
+using MongoDB.Bson;
+
+using System;
+
+public static class MongoDbExtensions {
+   public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsBsonArray;
+
+   public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
+         val.AsBsonArray : null;
+
+   public static string GetString(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsString;
+
+   public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsString ?
+         val.AsString : null;
+
+   public static int GetInt32(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsInt32;
+
+   public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
+         val.AsInt32 : null;
+
+   public static long GetInt64(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsInt64;
+
+   public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
+         val.AsInt64 : null;
+
+   public static double GetDouble(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsDouble;
+
+   public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsDouble ?
+         val.AsDouble : null;
+
+   public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
+      doc[elementName].ToUniversalTime();
+
+   public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
+         val.ToUniversalTime() : null;
+
+   public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
+      doc[elementName].ToUniversalTime();
+
+   public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
+         val.ToUniversalTime() : null;
+
+   public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsObjectId;
+
+   public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
+         val.AsObjectId : null;
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs

@@ -0,0 +1,55 @@
+namespace Bts.DataStream;
+
+using bts;
+
+using MongoDB.Bson;
+
+using System;
+using System.Collections.Generic;
+
+public static class MongoSchemaAdaptor {
+   public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
+      var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
+      foreach (var doc in docs) {
+         foreach (var el in doc) {
+            if (schemaAdaptor.Contains(el.Name)) {
+               continue;
+            }
+            switch (el.Value.BsonType) {
+               case BsonType.Array:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
+                  break;
+               case BsonType.String:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
+                  break;
+               case BsonType.Int32:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
+                  break;
+               case BsonType.Int64:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
+                  break;
+               case BsonType.Double:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
+                  break;
+               case BsonType.DateTime:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
+                  break;
+               case BsonType.Timestamp:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
+                  break;
+               case BsonType.ObjectId:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
+                  break;
+               case BsonType.Null:
+                  Log.Gen.warn(
+                     $"Ignoring column '{el.Name}' because at least one element is null");
+                  break;
+               default:
+                  throw new InvalidOperationException(
+                     $"Unsupported BsonType '{el.Value.BsonType}'");
+            }
+         }
+      }
+      return schemaAdaptor;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -6,6 +6,7 @@ namespace bts {
    using System;
    using System.Collections;
    using System.Collections.Generic;
+   using System.Linq;
 
    public class ProjectData {
       // schema from the base stream
@@ -147,12 +148,23 @@ namespace bts {
                mapped_to_orig_map.Add(mapped_col, orig_col);
                orig_to_mapped.Add(orig_col, mapped_col);
             } else {
-               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
             }
          }
          // make the new schema
          List<ColumnInfo> new_cols = new List<ColumnInfo>();
          foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               // if there's a hidden col, other columns will show by default, but if cols are being
+               // projected epxlicitely, than others wil be hidden
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
             if (removed_cols.Contains(ci.colName)) {
                continue;
             } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -160,8 +172,12 @@ namespace bts {
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
             } else {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
+               if (ci.colName == "id" || removed_cols.Count > 0) {
+                  // if there's a hidden col, other columns will show by default, but if cols are being
+                  // projected epxlicitely, than others wil be hidden
+                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+                  new_cols.Add(nc);
+               }
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());

src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs

@@ -1,25 +1,23 @@
 namespace BCG {
-   using mbts.spark;
-
    using Spark.Core.Collections;
 
-   using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
 
    [DataStreamUserRespresentation("oldproject")]
-   public class ProjectStream : ViewStream {
+   public class ProjectStream : ViewStream
+   {
       private ProjectHelper myHelper;
       private string myColExp, myTableName;
       private ViewStream myVs;
       private IDictionary<string, string> myDic;
-      private HighPerformanceTimer _timer = new();
       /// <summary>
       /// </summary>
       /// <param name="vs">stream to susbsribe to as the input stream</param>
       /// <param name="colExp">expression defining the column mapping</param>
-      public ProjectStream(ViewStream vs, string colExp) {
+      public ProjectStream (ViewStream vs, string colExp)
+      {
          myVs = vs;
          myColExp = colExp;
       }
@@ -29,10 +27,12 @@ namespace BCG {
       /// <param name="vs">stream to subscribe to as the input stream</param>
       /// <param name="colExp">expression defining the column mapping</param>
       /// <param name="tableName">name of the mapped table</param>
-      public ProjectStream(ViewStream vs, string colExp, string tableName) {
+      public ProjectStream (ViewStream vs, string colExp, string tableName)
+      {
          if (colExp == "!")
-            mySetRealSource(new ChangeTableNameStream(vs, tableName));
-         else {
+            mySetRealSource (new ChangeTableNameStream (vs, tableName));
+         else
+         {
             myVs = vs;
             myColExp = colExp;
             myTableName = tableName;
@@ -43,51 +43,57 @@ namespace BCG {
       /// </summary>
       /// <param name="vs">stream to subscribe to as the input stream</param>
       /// <param name="dic">dictionary mapping the aliased to the original</param>
-      public ProjectStream(ViewStream vs, IDictionary dic)
-         : this(vs, GenericDictionaryWrapper.Create<string, string>(dic)) { }
+      public ProjectStream (ViewStream vs, IDictionary dic)
+         : this(vs, GenericDictionaryWrapper.Create<string, string>(dic)) {}
 
-      public ProjectStream(ViewStream vs, IEnumerable<KeyValuePair<string, string>> dic) {
+      public ProjectStream (ViewStream vs, IEnumerable<KeyValuePair<string, string>> dic)
+      {
          myVs = vs;
          myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key);
       }
 
-      protected override void myOnFirstSubscriber() {
-         myVs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChange);
+      protected override void myOnFirstSubscriber()
+      {
+         myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
       }
 
-      protected override void myOnLastSubscriber() {
-         myVs.onChange -= new BCG.DataStream.ChangeDelegate(myHandleChange); ;
+      protected override void myOnLastSubscriber()
+      {
+         myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
       }
 
-      internal void myHandleChange(object newValue) {
+      internal void myHandleChange (object newValue)
+      {
          DataPacket dp = newValue as DataPacket;
-         if (myHelper != null) {
-            _timer.start();
+         if (myHelper != null)
+         {
             IBlueRowCollection dels, adds;
-            BlueDataUtility.getTrueDeletesAndAdds(dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
-            _timer.stop();
+            BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
             if (dels.Count + adds.Count > 0)
-               myFireEvent(new DataPacket(new ProjectedTable(dp.newRows, myHelper),
-                                            new ProjectedTable(adds, myHelper),
-                                             new ProjectedTable(dels, myHelper)
+               myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
+                                            new ProjectedTable (adds, myHelper),
+                                             new ProjectedTable (dels, myHelper)
                                           )
-                           );
-         } else {  //first time a callback is received
+                           );              
+         }
+         else
+         {  //first time a callback is received
             myHelper = myDic == null ?
-               new ProjectHelper(dp.schema, myColExp, myTableName) :
-               new ProjectHelper(dp.schema, myDic, myTableName);
+               new ProjectHelper (dp.schema, myColExp, myTableName) :
+               new ProjectHelper (dp.schema, myDic, myTableName);
 
             IBlueRowCollection temp;
-            myFireEvent(new DataPacket(temp = new ProjectedTable(dp.newRows, myHelper),
+            myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper),
                temp,
-               new BlueTable(temp.schema)
+               new BlueTable (temp.schema)
                )
                );
 
          }
       }
-      public void myHandleChangeTest(object newValue) {
-         myHandleChange(newValue);
+      public void myHandleChangeTest(object newValue)
+      {
+          myHandleChange(newValue);
       }
 
    }

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -0,0 +1,33 @@
+using BCG;
+
+namespace bts {
+    public class SchemaGuesserStream : ViewStream {
+
+        private IBlueRowCollection _rows = null;
+        private ViewStream _vs;
+        public SchemaGuesserStream(ViewStream vs) {
+            _vs = vs;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _on_data;
+        }
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _on_data;
+        }
+
+        private void _on_data(object obj) {
+            DataPacket dp = obj as DataPacket;
+            if (_rows == null) {
+                _rows = BlueDataUtility.get_typed_table(dp.newRows);
+            }
+            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+
+        }
+
+
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -1,115 +1,103 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Evaluator.SortExpression;
-
-
-// namespace BCG
-// {
-//    public class SortRow : IComparable, IBlueRow
-//    {
-//       private IBlueRow myRow;
-//       private IEnumerable myColumns;
-//       private IComparer myComparer;
-
-//       public SortRow (IBlueRow row, SortExpression sortExpression)
-//       {
-//          myRow = row;
-//          myColumns = sortExpression.columns;
-//       }
-
-//       public SortRow (IBlueRow row, IEnumerable columns)
-//       {
-//          myRow = row;
-//          myColumns = columns;
-//       }
-
-//       public SortRow (IBlueRow row, IComparer comparer)
-//       {
-//          myRow = row;
-//          myComparer = comparer;
-//       }
-
-//       public IBlueRow internal_row { get { return myRow; } }
-
-//       #region IComparable method
-//       public int CompareTo (object obj)
-//       {  
-//          IBlueRow otherRow = (IBlueRow)obj;
-//          if (myComparer == null)
-//          {
-//             foreach (SortExpression.ColumnCriteria cc in myColumns)
-//             {
-//                int retVal = myIsNull (this[cc.columnName]) ?
-//                   myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
-//                   myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
-//                retVal *= cc.ascending ? 1 : -1;
-//                if (retVal != 0)
-//                   return retVal;
-//             }
-//             return 0;
-//          }
-//          else
-//             return myComparer.Compare (this, otherRow);
-//       }
-//       #endregion
-
-//       #region IBlueRow members
-//       public IBlueTableSchema schema{get{return myRow.schema;}}
-//       #endregion
-
-//       #region ICollection members
-//       public int Count {get{return myRow.Count;}}
-
-//       public bool IsSynchronized {get{return false;}}
-
-//       public object SyncRoot {get{return this;}}
-
-//       public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
-//       #endregion
-
-//       #region IDictionary members
-//       public bool IsFixedSize {get{return true;}}
-
-//       public bool IsReadOnly {get{return true;}}
-
-//       /// <summary>
-//       /// The key can either be the string corrsponding to the column name or
-//       /// the order of the column
-//       /// </summary>
-//       public object this [object key]
-//       {
-//          get
-//          {
-//             if (key is string)
-//                return this[(string)key];
-//             return this[(int)key];
-//          }
-//          set {throw new NotSupportedException ("The IBlueRow is readonly");}
-//       }
-
-//       public object this [int index]   {get{return myRow[index];}}
-
-//       public object this [string colName] {get{return myRow[colName];}}
-
-//       public ICollection Keys{get{return myRow.Keys;}}
-
-//       public ICollection Values{get{return myRow.Values;}}
-
-//       public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
-
-//       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
-
-//       public bool Contains (object key){return myRow.Contains (key);}
-
-//       IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
-
-//       IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
-
-//       public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
-//       #endregion
-
-//       private bool myIsNull (object o) {return o == null || o is DBNull;}
-
-//    }
-// }
+using Bts.ColumnSortExpressions;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace BCG;
+
+public class SortRow : IComparable, IBlueRow {
+   readonly IBlueRow row;
+   readonly IEnumerable<ColumnSortExpression>? columns;
+   readonly IComparer? comparer;
+
+   public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) {
+      this.row = row;
+      this.columns = columns;
+   }
+
+   public SortRow(IBlueRow row, IComparer comparer) {
+      this.row = row;
+      this.comparer = comparer;
+   }
+
+   public IBlueRow internal_row { get { return row; } }
+
+   #region IComparable method
+   public int CompareTo(object? obj) {
+      if (obj is not IBlueRow otherRow) {
+         throw new ArgumentException(
+            $"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})",
+            nameof(obj));
+      }
+      if (comparer is not null) {
+         return comparer.Compare(this, otherRow);
+      }
+      return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0);
+      int Compare(ColumnSortExpression col) {
+         var otherVal = otherRow[col.Name];
+         int rv = myIsNull(this[col.Name]) ?
+            (myIsNull(otherVal) ? 0 : -1) :
+            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
+         rv *= col.IsAscending ? 1 : -1;
+         return rv;
+      }
+   }
+   #endregion
+
+   #region IBlueRow members
+   public IBlueTableSchema schema { get { return row.schema; } }
+   #endregion
+
+   #region ICollection members
+   public int Count { get { return row.Count; } }
+
+   public bool IsSynchronized { get { return false; } }
+
+   public object SyncRoot { get { return this; } }
+
+   public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
+   #endregion
+
+   #region IDictionary members
+   public bool IsFixedSize { get { return true; } }
+
+   public bool IsReadOnly { get { return true; } }
+
+   /// <summary>
+   /// The key can either be the string corrsponding to the column name or
+   /// the order of the column
+   /// </summary>
+   public object? this[object key] {
+      get => key is string str ? this[str] : this[(int)key];
+      set => throw new NotSupportedException("The IBlueRow is readonly");
+   }
+
+   public object? this[int index] => row[index];
+
+   public object? this[string colName] => row[colName];
+
+   public ICollection Keys => row.Keys;
+
+   public ICollection Values => row.Values;
+
+   public bool Contains(object key) => row.Contains(key);
+
+   void IDictionary.Add(object key, object? value) =>
+      throw new NotSupportedException("The IBlueRow is readonly");
+
+   void IDictionary.Clear() =>
+      throw new NotSupportedException("The IBlueRow is readonly");
+
+   IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); }
+
+   IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); }
+
+   public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+   #endregion
+
+   bool myIsNull(object? o) =>
+      o is null || o == null || o is DBNull;
+
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,129 +1,99 @@
-// using System;
-// using System.Collections;
-// using BCG.Evaluator.SortExpression;
+using Bts.ColumnSortExpressions;
 
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-// 	/// to merge the additions to the accumulates table
-// 	/// </summary>
-// 	public class SortStream : ViewStream
-// 	{
-//       private ViewStream myStream;
-//       private IEnumerable mySortCols;
-//       private IComparer myComparer;
-//       private DataPacket myLastFiredPacket;
-//       private DataPacket myLastRecievedPacket;
+namespace BCG;
 
-//       public SortStream (ViewStream stream, string sortExpression)
-//       {
-//          myStream = stream;
-//          mySetSortExpression (sortExpression);
-//       }
+/// <summary>
+/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+/// to merge the additions to the accumulates table
+/// </summary>
+public class SortStream : ViewStream {
+   ViewStream stream;
+   IList<ColumnSortExpression> sortCols;
+   IComparer? comparer;
+   DataPacket? lastFiredPacket;
+   DataPacket? lastReceivedPacket;
 
-//       public SortStream (ViewStream vs, string colName, bool isAsc)
-//       {
-//          myStream = vs;
-//          string sortExpression = isAsc ? "+" + colName : "-" + colName;
-//          mySetSortExpression (sortExpression);
-//       }
+   public SortStream(ViewStream stream, string sortExpression) {
+      this.stream = stream;
+      SortExpression = sortExpression;
+   }
 
-//       public SortStream (ViewStream vs, IComparer comparer)
-//       {
-//          myStream = vs;
-//          myComparer = comparer;
-//       }
+   public SortStream(ViewStream vs, string colName, bool isAsc) {
+      stream = vs;
+      SortExpression = isAsc ? "+" + colName : "-" + colName;
+   }
 
-//       public string sortExpression
-//       {
-//          set
-//          {
-//             myComparer = null;
-//             myLastFiredPacket = null;
-//             mySetSortExpression (value);   
-//             myChangeHandler (myLastRecievedPacket);
-//          }
-//       }
+   public SortStream(ViewStream stream, IComparer comparer) {
+      this.stream = stream;
+      this.comparer = comparer;
+      sortCols = Array.Empty<ColumnSortExpression>();
+   }
 
-//       public IComparer comparer
-//       {
-//          set
-//          {
-//             myComparer = value;
-//             myLastFiredPacket = null;
-//             myChangeHandler(myLastRecievedPacket);
-//          }
-//       }
+   [MemberNotNull(nameof(sortCols))]
+   public string SortExpression {
+      set {
+         comparer = null;
+         lastFiredPacket = null;
+         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
 
-//       private void mySetSortExpression (string sortExpression)
-//       {
-//          syntax parser = 
-//             new syntax();
-         
-//          mySortCols = ((SortExpression)
-//             parser.Parse (sortExpression)).columns;
-//       }
+   public IComparer Comparer {
+      set {
+         comparer = value;
+         lastFiredPacket = null;
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
 
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); 
-//       }
+   public void myChangeHandlerTest(object o) =>
+      myChangeHandler(o);
 
-//       protected override void myOnLastSubscriber()
-//       {
-//          myInit ();
-//          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); 
-//       }
+   [MemberNotNull(nameof(lastFiredPacket))]
+   internal void myChangeHandler(object o) {
+      lastReceivedPacket = (DataPacket)o;
+      // First callback, so we need to sort the whole table, this is the
+      // only time we will do that for all the other sorting operations would
+      // be sorting the increments and inserting them into the whole list
+      if (lastFiredPacket is null) {
+         lastFiredPacket = new() {
+            newRows = comparer is null ?
+               new SortedTable(lastReceivedPacket.newRows, sortCols) :
+               new SortedTable(lastReceivedPacket.newRows, comparer)
+         };
+         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
+         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
+         myFireEvent(lastFiredPacket.Clone());
+         return;
+      }
+      // Deletes
+      lastFiredPacket.deletedRows =
+         comparer == null ?
+         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
+         new SortedTable(lastReceivedPacket.deletedRows, comparer);
+      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
+      // Adds
+      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
+      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
+      myFireEvent(lastFiredPacket.Clone());
+   }
 
-//       internal void myChangeHandler (object o)
-//       {
-//          myLastRecievedPacket = (DataPacket)o;
-//          //first callback, so we need to sort the whole table, this is the
-//          //only time we will do that for all the other sorting operations would
-//          //be sorting the increments and inserting them into the whole list
-//          if (myLastFiredPacket == null)
-//          {
-//             myLastFiredPacket = new DataPacket ();
-//             myLastFiredPacket.newRows = 
-//                myComparer == null ?
-//                new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
-//                new SortedTable (myLastRecievedPacket.newRows, myComparer);
-//             myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//             myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-//             myFireEvent (myLastFiredPacket.Clone());
-//             return;
-//          }
-//          myHandleDeletes (myLastRecievedPacket);
-//          myHandleAdds (myLastRecievedPacket);
-//          myFireEvent (myLastFiredPacket.Clone());
-//       }
+   protected override void myOnFirstSubscriber() =>
+      stream.onChange += myChangeHandler;
 
-//       public void myChangeHandlerTest(object o)
-//       {
-//           myChangeHandler(o);
-//       }
-
-//       private void myHandleDeletes (DataPacket dp)
-//       {
-//          myLastFiredPacket.deletedRows = 
-//             myComparer == null ?
-//             new SortedTable (dp.deletedRows, mySortCols) :
-//             new SortedTable (dp.deletedRows, myComparer);
-//          ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
-//       }
-
-//       private void myHandleAdds (DataPacket dp)
-//       {
-//          myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
-//          ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
-//       }
-
-//       private void myInit ()
-//       {
-//          myLastFiredPacket = null;
-//          myLastRecievedPacket = null;
-//       }
-// 	}
-// }
+   protected override void myOnLastSubscriber() {
+      lastFiredPacket = null;
+      lastReceivedPacket = null;
+      stream.onChange -= myChangeHandler;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs

@@ -1,136 +1,108 @@
-// using System;
-// using System.Collections;
-// using BCG.Evaluator.SortExpression;
-
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Represents an IBlueRowCollection with the rows sorted according to the
-//    /// 
-//    /// </summary>
-//    public class SortedTable : IBlueRowCollection 
-//    {
-//       private ArrayList myRows = new ArrayList ();
-//       private Hashtable myIndex = new Hashtable ();
-//       private IBlueTableSchema mySchema;
-//       private IEnumerable mySortCols;
-//       private IComparer myComparer;
-
-//       #region SortedTable method(s)
-//       public void addRows (IBlueRowCollection rows)
-//       {
-//          //requires merger of the addedRows to the already existing ones
-//          if (rows.Count == 0)
-//             //at times there are calls with 0 additions, in that case
-//             //we ofcourse need no processing
-//             return;
-//          SortedTable addedRows = 
-//             myComparer == null ? new SortedTable (rows, mySortCols) :
-//                new SortedTable (rows, myComparer);
-//          int addedRowsIndex = 0;
-//          SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
-//          for (int i = 0; i < myRows.Count; i ++)
-//             if (newRow.CompareTo(myRows[i]) <= 0)
-//             {
-//                myRows.Insert (i, newRow);
-//                myIndex[newRow["id"]] = newRow;
-//                if (++addedRowsIndex == addedRows.Count)
-//                   return;
-//                newRow = addedRows.getRowByIndex (addedRowsIndex);
-//             }
-//          if (addedRowsIndex < addedRows.Count)
-//             for (int i = addedRowsIndex; i < addedRows.Count; i++)
-//             {
-//                newRow = addedRows.getRowByIndex (i);
-//                myRows.Add (newRow);
-//                myIndex[newRow["id"]] = newRow;
-//             }
-//       }
-
-//       public void deleteRows (IBlueRowCollection deletedRows)
-//       {
-//          foreach (IBlueRow deletedRow in deletedRows)
-//          {
-//             IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
-//             myRows.Remove (row);
-//             myIndex.Remove (row["id"]);
-//          }
-//       }
-
-//       public SortRow getRowByIndex (int index)  {return (SortRow)myRows[index];}
-//       #endregion
-
-//       #region Constructor(s)
-//       /// <summary>
-//       /// Sorts the rows using sortExpression as the sorting criteria
-//       /// </summary>
-//       /// <param name="rows">
-//       /// Rows to be inserted into the table
-//       /// </param>
-//       /// <param name="sortExpression">
-//       /// Sorting criteria
-//       /// </param>
-//       public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
-//       {
-//          mySortCols = sortExpression.columns;
-//          myConstructorHelper (rows);
-//       }
-
-//       public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
-//       {
-//          mySortCols = sortCols;
-//          myConstructorHelper (rows);
-//       }
-
-//       public SortedTable (IBlueRowCollection rows, IComparer comparer)
-//       {
-//          myComparer = comparer;
-//          myConstructorHelper (rows);
-//       }
-
-//       private void myConstructorHelper (IBlueRowCollection rows)
-//       {
-//          mySchema = rows.schema;
-//          foreach (IBlueRow row in rows)
-//          {
-//             SortRow newRow = 
-//                myComparer == null ?
-//                new SortRow (row, mySortCols) : new SortRow (row, myComparer);
-//             myRows.Add (newRow);
-//             myIndex[row["id"]] = newRow;
-//          }
-//          myRows.Sort ();
-//       }
-
-//       #endregion
-
-//       #region IBlueRowCollections members
-//       public IBlueRow this[int id] 
-//       {
-//          get
-//          {
-//             if (myIndex[id] == null)
-//                throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
-//             return (IBlueRow)myIndex[id];
-//          }
-//       }
-//       public IBlueTableSchema schema {get{return mySchema;}}
-
-//       public bool containsID (int id)  {return myIndex[id] != null;}
-
-//       #endregion
-
-//       #region ICollection members
-//       public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
-
-//       public bool IsSynchronized {get{return false;}}
-
-//       public object SyncRoot {get{return this;}}
-
-//       public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
-
-//       public int Count{get{return myRows.Count;}}
-//          #endregion
-//    }
-
-// }
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+using Bts.ColumnSortExpressions;
+
+namespace BCG;
+
+/// <summary>
+/// Represents an IBlueRowCollection with the rows sorted according to the
+/// </summary>
+public class SortedTable : IBlueRowCollection {
+   readonly List<SortRow> rows = new(0);
+   readonly Dictionary<int, IBlueRow> rowIndex = new(0);
+   readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>();
+   readonly IComparer? comparer;
+
+   /// <summary>
+   /// Sorts the rows using sortExpression as the sorting criteria
+   /// </summary>
+   /// <param name="rows">Rows to be inserted into the table</param>
+   /// <param name="sortCols">Sorting criteria</param>
+   public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) {
+      this.sortCols = sortCols;
+      init(rows);
+   }
+
+   public SortedTable(IBlueRowCollection rows, IComparer comparer) {
+      this.comparer = comparer;
+      init(rows);
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; private set; }
+   public object SyncRoot => this;
+   public IBlueRow this[int id] => rowIndex[id]
+      ?? throw new BlueDataException(
+         $"ID '{id}' does not exist in table '{schema?.tableName}'");
+
+   public void addRows(IBlueRowCollection rows) {
+      // Requires merger of the addedRows to the already existing ones
+      if (rows.Count == 0) {
+         // At times there are calls with 0 additions, in that case
+         // we of course need no processing
+         return;
+      }
+      SortedTable addedRows =
+         comparer == null ? new SortedTable(rows, sortCols) :
+            new SortedTable(rows, comparer);
+      int addedRowsIndex = 0;
+      SortRow newRow = addedRows.getRowByIndex(addedRowsIndex);
+      for (int i = 0; i < this.rows.Count; i++) {
+         if (newRow.CompareTo(this.rows[i]) <= 0) {
+            this.rows.Insert(i, newRow);
+            rowIndex[(int)newRow["id"]] = newRow;
+            if (++addedRowsIndex == addedRows.Count) {
+               return;
+            }
+            newRow = addedRows.getRowByIndex(addedRowsIndex);
+         }
+      }
+      if (addedRowsIndex < addedRows.Count) {
+         for (int i = addedRowsIndex; i < addedRows.Count; i++) {
+            newRow = addedRows.getRowByIndex(i);
+            this.rows.Add(newRow);
+            rowIndex[(int)newRow["id"]] = newRow;
+         }
+      }
+   }
+
+   public void deleteRows(IBlueRowCollection deletedRows) {
+      foreach (IBlueRow deletedRow in deletedRows) {
+         var row = rowIndex[(int)deletedRow["id"]!] as SortRow
+            ?? throw new InvalidOperationException("Index did not return IBlueRow");
+         rows.Remove(row);
+         rowIndex.Remove((int)row["id"]!);
+      }
+   }
+
+   public SortRow getRowByIndex(int index) =>
+      rows[index]!;
+
+   public bool containsID(int id) =>
+      rowIndex[id] is not null;
+
+   public IEnumerator GetEnumerator() =>
+      rows.GetEnumerator();
+
+   public void CopyTo(Array array, int index) {
+      for (int i = 0; i < Count; i++) {
+         array.SetValue(rows[i], index + i);
+      }
+   }
+
+   [MemberNotNull(nameof(schema))]
+   void init(IBlueRowCollection rows) {
+      schema = rows.schema;
+      foreach (IBlueRow row in rows) {
+         SortRow newRow = comparer is null ?
+            new(row, sortCols) : new(row, comparer);
+         this.rows.Add(newRow);
+         rowIndex[(int)row["id"]!] = newRow;
+      }
+      this.rows.Sort();
+   }
+}

src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/MutableStream.cs

@@ -22,9 +22,12 @@ namespace BCG
       /// <param name="rows"></param>
       public void loadCollection (IBlueRowCollection rows)
       {
-         myFireEvent (new DataPacket (rows, new BlueTable(rows), 
-            myLastValue != null ?
-            new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
+         myFireEvent(new DataPacket(
+            rows,
+            new BlueTable(rows), 
+            myLastValue is DataPacket dp
+               ? new BlueTable(dp.newRows)
+               : new BlueTable(rows.schema)));
       }
 
       /// <summary>

src/DataStreamSuite/DataStream/Primitives/ThreadSafe/VSDBMarshallerStream.cs

@@ -123,7 +123,7 @@ namespace BCG
          public Guid subscription;
       }
       private ThreadSafeQueue<Update> _queued_updates;
-      TargetDoWorkRelay _do_work_relay;
+      TargetDoWorkRelay? _do_work_relay;
       Guid _current_source_subscription;
       Guid _current_target_subscription;
       int _source_thread;

src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs

@@ -1,29 +1,71 @@
 using System;
 
-namespace BCG
-{
-   public class MSecondStream : TimeUnitStream 
-   {
-      public MSecondStream (int intervalInms, int size)
-         : base (DateTime.Now, new TimeSpan (0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance)
-      {}
+using BCG.Utilities.General;
 
-      protected override DateTime myRoundTime (DateTime dt) {return dt;}
+using bts;
+
+namespace BCG {
+   [DataStreamUserRespresentation("milliseconds")]
+   public class MillisecondStream : ViewStream, IWorker {
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private readonly TimeSpan _period;
+      private readonly int _size;
+      private readonly ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema;
+      private bts.CircularBufferTable _table;
+
+      public MillisecondStream(int interval_msecs, int size)
+      : this(ViewStreamDatabase.instance, interval_msecs, size) {
+      }
+
+      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
+         _vsd = vsd;
+         _period = TimeSpan.FromMilliseconds(interval_msecs);
+         _size = size;
+         _schema = BlueDataUtility.form_schema("Milliseconds",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("time", TypeCode.DateTime),
+               Tuple.Create("display_time", TypeCode.String));
+         _table = new CircularBufferTable(_schema, _size);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private int _id = 0;
+      public void doWork() {
+         if (DateTime.Now - _last_fire_time >= _period) {
+            _last_fire_time = DateTime.Now;
+            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
+            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
+            _table.add_row(nr);
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+   }
+   public class MSecondStream : TimeUnitStream {
+      public MSecondStream(int intervalInms, int size)
+         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
+
+      protected override DateTime myRoundTime(DateTime dt) { return dt; }
    }
 
-   public class DisplayableMillisecondStream : ViewStream, IStreamMapper
-   {
-      private IBlueTableSchema mySchema = new BlueTableSchema ("ms",
-                                                               new ColumnInfo ("id", 0, typeof(int)),
-                                                               new ColumnInfo ("time", 1, typeof(string)));
-      public DisplayableMillisecondStream (int intervalInMs, int sz)
-      {
-         mySetRealSource (new MapStream (new MSecondStream (intervalInMs, sz), this));
+   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
+                                                               new ColumnInfo("id", 0, typeof(int)),
+                                                               new ColumnInfo("time", 1, typeof(string)));
+      public DisplayableMillisecondStream(int intervalInMs, int sz) {
+         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
       }
       #region IStreamMapper Members
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
          return mySchema;
       }
 
@@ -31,15 +73,15 @@ namespace BCG
 
       #region IDynamicStreamMapper Members
 
-      public IBlueRow map (IBlueRow row)
-      {
-         return new BlueRow (mySchema, row["id"], myToString ((DateTime)row["time"]));
+      public IBlueRow map(IBlueRow row) {
+         DateTime time = row.get_datetime("time");
+         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+         return new BlueRow(mySchema, row["id"], time_txt);
       }
 
       #endregion
 
-      private string myToString (DateTime dt)
-      {
+      private string myToString(DateTime dt) {
          string sep = ":";
          return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
       }

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,49 +1,43 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class TopStream : MethodDrivenStream
-//    {
-//       private int myTopCount;
+namespace BCG {
+    public class TopStream : MethodDrivenStream {
+        private int myTopCount;
 
-//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
+        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
 
-//       public TopStream (ViewStream stream, 
-//          int topCount, 
-//          string sortExpression) 
-//          :
-//          this (sortExpression == null? 
-//          stream :
-//          new SortStream (stream, sortExpression), topCount)
-//       {}
+        public TopStream(ViewStream stream,
+           int topCount,
+           string sortExpression)
+           :
+           this(sortExpression == null ?
+           stream :
+           new SortStream(stream, sortExpression), topCount) { }
 
-//       protected override void myOnTableClear () 
-//       {
-//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//       }
+        protected override void myOnTableClear() {
+            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+        }
 
-//       protected override void myHandleFirstCall () 
-//       {
-//          myLastFiredPacket = new DataPacket ();
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
-//       }
+        protected override void myHandleFirstCall() {
+            myLastFiredPacket = new DataPacket();
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+        }
 
-//       protected override void myHandleIncrementalAdd ()     {}
+        protected override void myHandleIncrementalAdd() { }
 
-//       protected override void myHandleIncrementalDelete()   
-//       {
-//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
-//             oldRows,
-//             myLastFiredPacket.schema.columnNames,
-//             out myLastFiredPacket.deletedRows,
-//             out myLastFiredPacket.addedRows);
-//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-//       }
-//    }
-// }
+        protected override void myHandleIncrementalDelete() {
+            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+               oldRows,
+               myLastFiredPacket.schema.columnNames,
+               out myLastFiredPacket.deletedRows,
+               out myLastFiredPacket.addedRows);
+            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -1,8 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.Text;
 using mbts;
-using mbts.spark;
 
 namespace BCG
 {
@@ -183,7 +181,6 @@ namespace BCG
    /// </summary>
    public class MutableAppendStream : ViewStream
    {
-      HighPerformanceTimer _timer = new HighPerformanceTimer();
       private bool mySubscriptionActive = false;
       private MutableAppendTable myTable;
       private int myMovingIndex;
@@ -272,7 +269,6 @@ namespace BCG
       private void myHandleStreamChanges(object newValue)
       {         
          DataPacket dp = newValue as DataPacket;
-         _timer.start();
          myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
          BlueTable adds = new BlueTable(myTable.schema),
                      dels = new BlueTable(myTable.schema);
@@ -287,7 +283,6 @@ namespace BCG
             IBlueRow nr = myTable.addRow(r, index);
             adds.addRow(nr);
          }
-         _timer.stop();
          myFireEvent(new DataPacket(myTable, adds, dels));
       }
    }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -348,6 +348,64 @@ namespace BCG {
          return new BlueTableSchema(table_name, cis.ToArray());
       }
 
+      public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
+         // just use one row to guess
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         foreach (IBlueRow r in rows) {
+            foreach (var ci in r.schema.columnInfo) {
+               if (ci.colType != typeof(System.String)) {
+                  cis.Add(ci);
+                  continue;
+               }
+               var (type, _) = guess_type(r[ci.colName] as string);
+               cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
+            }
+            return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         }
+         return null;
+      }
+
+      // converts a row where all columns are string types to a schema where the best types are guesed
+      public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
+         IBlueTableSchema typed_schema = get_typed_schema(string_rows);
+         var rv = new BlueTable(typed_schema);
+         foreach (IBlueRow r in string_rows) {
+            var typed_row = get_typed_row(r, typed_schema);
+            rv.addRow(typed_row);
+         }
+         return rv;
+
+         static IBlueRow get_typed_row(IBlueRow string_row, IBlueTableSchema typed_schema) {
+            var rv = new MutableBlueRow(typed_schema);
+            foreach (var ci in typed_schema.columnInfo) {
+               var typed_val = convert(ci.colType, string_row[ci.colName].ToString());
+               rv.setValue(ci.colName, typed_val);
+            }
+            return rv;
+         }
+
+         static object convert(Type t, string value) {
+            // Attempt to convert the string to the specified type
+            try {
+               return Convert.ChangeType(value, t);
+            } catch (Exception) {
+               if (t == typeof(double)) {
+                  // some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
+                  string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
+                  cleaned_val = cleaned_val.Replace('−', '-');
+                  double rv_d;
+                  if (Double.TryParse(cleaned_val, out rv_d))
+                     return rv_d;
+                  else
+                     return null;
+               }
+               return null;
+               // Handle conversion failure gracefully
+               // throw new ArgumentException("Failed to convert the string to the specified type.");
+            }
+         }
+      }
+
       /// <summary>
       /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
       /// be alphabetic or not
@@ -1570,6 +1628,13 @@ namespace BCG {
          return sb.ToString();
       }
 
+      public static string id_string(IBlueRowCollection table) {
+         string rv = String.Empty;
+         foreach (IBlueRow r in table)
+            rv += $"{r["id"]} ";
+         return rv;
+      }
+
       public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
          BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
          return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
@@ -1718,6 +1783,18 @@ namespace BCG {
          #endregion
       }
 
+      public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
+         if (rows == null || rows.Count() == 0)
+            return null;
+         IBlueRow max_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            int c = compare_rows(r, max_row, cols_to_compare);
+            if (c > 0)
+               max_row = r;
+         }
+         return max_row;
+      }
+
       public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
          foreach (KeyValuePair<string, bool> p in sort_map) {
             int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
@@ -2152,7 +2229,10 @@ namespace BCG {
          }
       }
 
-      public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) {
+      public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) =>
+         createRowsFromCSVFileWithoutTypes(fileAddress, out var _);
+
+      public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress, out IBlueTableSchema schema) {
          StreamReader sr = new StreamReader(fileAddress);
          string cols = sr.ReadLine();
          string[] columns = cols.Split(',');
@@ -2160,11 +2240,11 @@ namespace BCG {
          ci[0] = new ColumnInfo("id", 0, typeof(int));
          for (int i = 1; i < ci.Length; i++)
             ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
-         IBlueTableSchema bts = new BlueTableSchema(ci, "csvTable");
-         BlueTable result = new BlueTable(bts);
+         schema = new BlueTableSchema(ci, "csvTable");
+         BlueTable result = new BlueTable(schema);
          int id = 0;
          while (!sr.EndOfStream)
-            result.addRow(createRowFromCSVFileRow(sr.ReadLine(), bts, id++));
+            result.addRow(createRowFromCSVFileRow(sr.ReadLine(), schema, id++));
          sr.Close();
          return result;
       }

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionEqualityComparer.cs

@@ -0,0 +1,35 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public class BlueRowCollectionEqualityComparer : IEqualityComparer<IBlueRowCollection?> {
+   readonly IEqualityComparer<IBlueRow> rowComparer = BlueRowEqualityComparer.Instance;
+
+   public static BlueRowCollectionEqualityComparer Instance { get; } = new();
+
+   public bool Equals(IBlueRowCollection? x, IBlueRowCollection? y) {
+      if (x is null) {
+         return y is null;
+      }
+      if (x.Count != y?.Count) {
+         return false;
+      }
+      return x.schema.columnInfo.SequenceEqual(y.schema.columnInfo)
+         && x.Cast<IBlueRow>().OrderBy(r => r.Id()).SequenceEqual(
+            y.Cast<IBlueRow>().OrderBy(r => r.Id()),
+            rowComparer);
+   }
+
+   public int GetHashCode([DisallowNull] IBlueRowCollection rows) {
+      var hash = new HashCode();
+      hash.Add(rows.schema.columnInfo.Length);
+      hash.Add(rows.Count);
+      return hash.ToHashCode();
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEqualityComparer.cs

@@ -0,0 +1,24 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public class BlueRowEqualityComparer : IEqualityComparer<IBlueRow> {
+   public static BlueRowEqualityComparer Instance { get; } = new();
+
+   public bool Equals(IBlueRow x, IBlueRow y) =>
+      x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) &&
+      x.Values.Cast<object>().SequenceEqual(y.Values.Cast<object>());
+
+   public int GetHashCode([DisallowNull] IBlueRow obj) {
+      var hash = new HashCode();
+      foreach (var v in obj.Values) {
+         hash.Add(v?.GetHashCode() ?? 0);
+      }
+      return hash.ToHashCode();
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs

@@ -1,8 +1,15 @@
+using Bts.DataStreamCore;
+
+using Spark.DataStream;
+
 using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Data;
+using System.Diagnostics.CodeAnalysis;
+
 //using BCG.Utilities;
+
 using System.Linq;
 
 namespace BCG {
@@ -10,7 +17,7 @@ namespace BCG {
    /// Anologue of the dotNet DataTable.
    /// </summary>
    [Serializable]
-   public class BlueTable : IBlueRowCollection {
+   public class BlueTable : IBlueRowCollection, IDictionary<int, IBlueRow> {
       /// <summary>
       /// Hashtable indexed on "id" to speed up removes
       /// </summary>
@@ -86,6 +93,12 @@ namespace BCG {
                throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
             return (IBlueRow)row;
          }
+         set {
+            if ((int)value["id"] != id) {
+               throw new ArgumentException("id arg not equal to row's actual id", nameof(id));
+            }
+            changeRow(value);
+         }
       }
 
       /// <summary>
@@ -202,6 +215,8 @@ namespace BCG {
 
       #endregion
 
+      public IDictionary<int, IBlueRow> AsDictionary() => this;
+
       #region ICollection members
       public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
       { return myIdIndex.Values.GetEnumerator(); }
@@ -214,5 +229,55 @@ namespace BCG {
 
       public virtual int Count { get { return myIdIndex.Count; } }
       #endregion
+
+      #region IDictionary<int, IBlueRow> members
+      public void Add(int key, IBlueRow value) {
+         throw new NotImplementedException();
+      }
+
+      bool IDictionary<int, IBlueRow>.ContainsKey(int key) =>
+         containsID(key);
+
+      bool IDictionary<int, IBlueRow>.Remove(int key) =>
+         deleteRow(key);
+
+      bool IDictionary<int, IBlueRow>.TryGetValue(
+         int key, [MaybeNullWhen(false)] out IBlueRow value) =>
+         tryGetValue(key, out value);
+
+      void ICollection<KeyValuePair<int, IBlueRow>>.Add(KeyValuePair<int, IBlueRow> item) =>
+         addRow(item.Value);
+
+      void ICollection<KeyValuePair<int, IBlueRow>>.Clear() =>
+         clear();
+
+      bool ICollection<KeyValuePair<int, IBlueRow>>.Contains(KeyValuePair<int, IBlueRow> item) =>
+         tryGetValue(item.Key, out var r)
+         && BlueRowEqualityComparer.Instance.Equals(r, item.Value);
+
+      void ICollection<KeyValuePair<int, IBlueRow>>.CopyTo(
+         KeyValuePair<int, IBlueRow>[] array, int arrayIndex) {
+         foreach (DictionaryEntry de in myIdIndex) {
+            array[arrayIndex++] = new((int)de.Key, (IBlueRow)de.Value);
+         }
+      }
+
+      bool ICollection<KeyValuePair<int, IBlueRow>>.Remove(KeyValuePair<int, IBlueRow> item) =>
+         deleteRow(item.Key);
+
+      IEnumerator<KeyValuePair<int, IBlueRow>> IEnumerable<KeyValuePair<int, IBlueRow>>.GetEnumerator() {
+         foreach (DictionaryEntry de in myIdIndex) {
+            yield return new((int)de.Key, (IBlueRow)de.Value);
+         }
+      }
+
+      ICollection<int> IDictionary<int, IBlueRow>.Keys =>
+         GenericCollectionWrapper.Create<int>(myIdIndex.Keys);
+
+      ICollection<IBlueRow> IDictionary<int, IBlueRow>.Values =>
+         GenericCollectionWrapper.Create<IBlueRow>(myIdIndex.Values);
+
+      bool ICollection<KeyValuePair<int, IBlueRow>>.IsReadOnly => false;
+      #endregion
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs

@@ -0,0 +1,98 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+// I would have named this ImmutableBlueTable, but that was already taken.
+public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
+   readonly ImmutableDictionary<int, IBlueRow> rows;
+
+   public BlueTableImmutable(IBlueTableSchema schema)
+      : this(schema, ImmutableDictionary<int, IBlueRow>.Empty) { }
+
+   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, IBlueRow> rows) {
+      this.schema = schema;
+      this.rows = rows;
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; }
+   public object SyncRoot => this;
+
+   public IBlueRow this[int id] => rows[id];
+
+   public bool containsID(int id) =>
+      rows.ContainsKey(id);
+
+   public void CopyTo(Array array, int index) {
+      foreach (var r in rows.Values) {
+         array.SetValue(r, index++);
+      }
+   }
+
+   public IEnumerator GetEnumerator() =>
+      rows.Values.GetEnumerator();
+
+   public DataPacketBuilder ToDataPacketBuilder() =>
+      new(this);
+
+   public class DataPacketBuilder {
+      readonly IBlueTableSchema schema;
+      readonly ImmutableDictionary<int, IBlueRow>.Builder table;
+      readonly ImmutableDictionary<int, IBlueRow>.Builder added =
+         ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
+      readonly ImmutableDictionary<int, IBlueRow>.Builder removed =
+         ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
+
+      public DataPacketBuilder(BlueTableImmutable table) {
+         schema = table.schema;
+         this.table = table.rows.ToBuilder();
+      }
+
+      public void AddOrUpdate(IBlueRow row, IEqualityComparer<IBlueRow>? comparer = null) {
+         var id = row.Id();
+         if (table.TryGetValue(id, out var orig)) {
+            if (comparer?.Equals(row, orig) ?? false) {
+               // No op: This row was already in the table (according to 'comparer').
+               // Remove it from 'added' and 'removed'.
+               added.Remove(id);
+               removed.Remove(id);
+               return;
+            }
+            // Change op: A row with this id was already in the table,
+            // but its values have changed (according to 'comparer').
+            removed[id] = orig;
+         }
+         added[id] = row;
+      }
+
+      public void Remove(int id) {
+         added.Remove(id);
+         if (table.TryGetValue(id, out var orig)) {
+            removed[id] = orig;
+         }
+      }
+
+      public DataPacket ToDataPacket(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new(newRows, addedRows, removedRows);
+      }
+
+      public BlueTableImmutable ToImmutable() =>
+         new(schema, table.ToImmutable());
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs

@@ -0,0 +1,87 @@
+#nullable enable
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueTableSchemaAdaptor<T> : IMutableBlueRowFactory<T> {
+   readonly string name;
+   readonly List<ColumnAdaptor> columns = new();
+   BlueTableSchema? schema;
+
+   public BlueTableSchemaAdaptor(string name) =>
+      this.name = name;
+
+   public IBlueTableSchema Schema =>
+      schema ??= new BlueTableSchema(columns.WrapSelectorRO(c => c.Info).ToArray(), name);
+
+   public bool Contains(string name) =>
+      columns.Any(c => c.Info.colName == name);
+
+   public void AddColumn(string name, Func<T, int> getValue) =>
+      AddColumn(name, r => getValue(r), typeof(int));
+
+   public void AddColumn(string name, Func<T, int?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int));
+
+   public void AddColumn(string name, Func<T, long> getValue) =>
+      AddColumn(name, r => getValue(r), typeof(long));
+
+   public void AddColumn(string name, Func<T, long?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long));
+
+   public void AddColumn(string name, Func<T, double> getValue) =>
+      AddColumn(name, r => getValue(r), typeof(double));
+
+   public void AddColumn(string name, Func<T, double?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double));
+
+   public void AddColumn(string name, Func<T, DateTime> getValue) =>
+      AddColumn(name, r => getValue(r), typeof(DateTime));
+
+   public void AddColumn(string name, Func<T, DateTime?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime));
+
+   public void AddColumn(string name, Func<T, string?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string));
+
+   public void AddColumn(string name, Func<T, object?> getValue, Type dataType) {
+      if (!ColumnInfo.isAllowedType(dataType)) {
+         throw new ArgumentException(
+            $"'{dataType}' is not a supported column type", nameof(dataType));
+      }
+      // Invalidate cached schema (set it to null)
+      schema = null;
+      columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r)));
+   }
+
+   public IBlueRow GetBlueRow(T item) =>
+      GetMutableBlueRow(item);
+
+   public MutableBlueRow GetMutableBlueRow(T item) {
+      var mbr = new MutableBlueRow(Schema);
+      foreach (var col in columns) {
+         mbr.setValue(col.Info.colNo, col.GetValue(item));
+      }
+      return mbr;
+   }
+
+   class ColumnAdaptor {
+      readonly Func<T, object?> getValue;
+
+      public ColumnAdaptor(ColumnInfo info, Func<T, object?> getValue) {
+         Info = info;
+         this.getValue = getValue;
+      }
+
+      public ColumnInfo Info { get; }
+
+      public object? GetValue(T row) =>
+         getValue(row);
+   }
+}