refactoring onix options

This reverts commit 4f05b95bd0.

.devcontainer/Dockerfile

@@ -1,125 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Note installations that take a while (more than 10s)
-
-RUN apt-get update
-RUN apt-get install -y ca-certificates
-RUN apt-get install -y ccache
-RUN apt-get install -y curl
-RUN apt-get install -y fish
-RUN apt-get install -y gdb
-# git-all: ~250s
-RUN apt-get install -y git-all
-RUN apt-get install -y gnupg
-# liblttng-ust-dev: ~30s
-RUN apt-get install -y liblttng-ust-dev
-RUN apt-get install -y libssh2-1-dev
-RUN apt-get install -y locales
-RUN apt-get install -y ninja-build
-# openssh-server: ~40s
-RUN apt-get install -y openssh-server
-RUN apt-get install -y pkg-config
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y sudo
-RUN apt-get install -y tzdata
-RUN apt-get install -y vim
-RUN apt-get install -y zip
-
-# Python: ~30s
-RUN apt-get install -y \
-   python3-pip \
-   python3-venv \
-   && python3 -m pip install --upgrade pip
-
-# GCC
-ARG gcc_major=13
-RUN \
-   add-apt-repository ppa:ubuntu-toolchain-r/test \
-   && apt-get update \
-   && apt-get install -y g++-$gcc_major
-
-# Clang
-ARG clang_major=17
-RUN \
-   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
-   | tee -a "/etc/apt/sources.list.d/llvm.list" \
-   && apt-get update \
-   && apt-get install -y \
-   clang-$clang_major \
-   clang-format-$clang_major \
-   clangd-$clang_major
-
-# CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
-
-# Dotnet
-RUN apt-get update && apt-get install -y dotnet7
-
-# Node
-ARG node_major=18
-RUN mkdir -p /etc/apt/keyrings \
-   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
-   | tee /etc/apt/sources.list.d/nodesource.list \
-   && apt-get update \
-   && apt-get install -y nodejs
-
-# Emumba (deps)
-RUN apt-get update && apt-get -y install rapidjson-dev
-# Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
-
-# Pcap
-RUN apt-get update && apt-get install -y libpcap0.8-dev
-
-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
-# Set up default executables (update-alternatives)
-RUN \
-   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
-   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
-   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
-   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
-
-# Add compiler symlinks for ccache 
-RUN \
-   ln -s /usr/bin/ccache /usr/local/bin/clang; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-   groupadd -g "$docker_uid" "$docker_user" \
-   && useradd \
-   -m \
-   -u "$docker_uid" \
-   -g "$docker_uid" \
-   -G sudo \
-   --password "$(openssl passwd -1 "$docker_pass")" \
-   "$docker_user"

.devcontainer/devcontainer.json

@@ -1,50 +0,0 @@
-{
-  "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
-  "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
-  },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
-  "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
-  "updateRemoteUserUID": true,
-  "shutdownAction": "none",
-  // Note: If running multiple containers on the same host, each must use a
-  // unique set of ports. Usually we just add 1000 to each port as needed. For
-  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
-  "runArgs": [
-    "--shm-size",
-    "2gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345"
-  ],
-  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
-  ],
-  "customizations": {
-    "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
-      "extensions": [
-        "eamodio.gitlens",
-        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
-      ]
-    }
-  }
-}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.gitattributes

@@ -1,3 +0,0 @@
-*.cmd text=auto eol=crlf
-*.cs text=auto eol=crlf
-*.sh text=auto eol=lf

.gitignore

@@ -8,9 +8,7 @@ cmake-build-debug/
 .idea/
 .dir-locals.el
 *.pyc
-**.vscode/*
-!.vscode/launch.json
-!.vscode/tasks.json
+**.vscode
 .vs
 obj/
 *.csproj.user
@@ -49,13 +47,13 @@ trace_data/
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
 /src/server/sparkd/cmds/spartan/analysis/data
-.tmp/
-.venv*/
+*.tmp/*
+/.venv*/*
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-.devcontainer/
+**/devcontainer.json
 *.Generated.cs
 *.lst

.vscode/launch.json

@@ -1,158 +0,0 @@
-{
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": ".NET Core Attach",
-      "type": "coreclr",
-      "request": "attach"
-    },
-    {
-      "name": "DataStreamPublisher",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
-      "cwd": "build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "DataStreamQueryAnalyzer",
-      "type": "coreclr",
-      "request": "launch",
-      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-        "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "PubSubServer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
-      "cwd": "build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Parameterized)",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build Spark (Debug)",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "args": [
-          "--sparkd",
-          "${input:sparkConfigServer}"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sparkd",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
-      "program": "",
-      "linux": {
-        "program": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "build/sparkd-debug-clang/server",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    }
-  ],
-  "inputs": [
-    {
-      "id": "dataStreamPublisherConfig",
-      "type": "pickString",
-      "description": "Publisher Configuration",
-      "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
-      ],
-      "default": "MarketDataPublisher",
-    },
-    {
-      "id": "pubSubServerConfig",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
-      ],
-      "default": "PubSubServer",
-    },
-    {
-      "id": "sparkConfigServer",
-      "type": "promptString",
-      "description": "Config Server URL",
-      "default": "http://localhost:8000/config",
-    },
-  ]
-}

.vscode/tasks.json

@@ -1,828 +0,0 @@
-{
-  "version": "2.0.0",
-  "tasks": [
-    {
-      "label": "Build Api Server",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Api Server (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Api Server (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--preset",
-          "${input:sparkdCmakePreset}",
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "${input:sparkdCompiler}"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug | Gcc)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "gcc"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug | Clang)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "clang"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "${input:sparkdCompiler}"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release | Gcc)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "gcc"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release | Clang)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "clang"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Edit DataStream Config",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "editor",
-        "args": [
-          "${env:HOME}/.local/share/DataStream/conf/spark.conf"
-        ],
-      },
-      "windows": {
-        "command": "vim",
-        "args": [
-          "${env:APPDATA}/bts/DataStream/conf/spark.conf"
-        ],
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "shared",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-    {
-      "label": "Edit Spark Config",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "vim",
-        "args": [
-          "${env:APPDATA}/bts/conf/spark.conf"
-        ],
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "dedicated",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher",
-      "dependsOn": "Build DataStream",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher (Release)",
-      "dependsOn": "Build DataStream (Release)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer",
-      "dependsOn": "Build DataStream",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Debug | Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Release)",
-      "dependsOn": "Build DataStream (Release)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Release | Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Spark",
-      "dependsOn": "Build Spark",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Release)",
-      "dependsOn": "Build Spark (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark Config Query",
-      "dependsOn": "Build Spark",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "Spark.Console",
-      },
-      "windows": {
-        "command": "Spark.Console.exe",
-      },
-      "args": [
-        "config",
-        "--query",
-        "${input:sparkConfigQuery}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Sparkd Config Query",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "config",
-        "--query",
-        "${input:sparkdConfigQuery}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/logging/show_last_run.cmd",
-        "args": [
-          "DataStream",
-          "DataStreamQueryAnalyzer.exe",
-          "'Starting DataStreamQueryAnalyzer'"
-        ]
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "shared",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-    {
-      "label": "View PubSubServer Logs (Last Run)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/logging/show_last_run.sh",
-        "args": [
-          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
-          "Starting PubSubServer"
-        ]
-      },
-      "windows": {
-        "command": "tools/logging/show_last_run.cmd",
-        "args": [
-          "DataStream",
-          "PubSubServer.exe",
-          "Starting PubSubServer"
-        ]
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "shared",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-  ],
-  "inputs": [
-    {
-      "id": "buildConfiguration",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "Debug",
-        "Release",
-      ],
-      "default": "Debug",
-    },
-    {
-      "id": "sparkdCmakePreset",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "sparkd-debug-clang",
-        "sparkd-debug-gcc",
-        "sparkd-release-clang",
-        "sparkd-release-gcc",
-      ],
-      "default": "sparkd-debug-clang",
-    },
-    {
-      "id": "sparkdCompiler",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "clang",
-        "gcc",
-      ],
-      "default": "sparkd-debug-clang",
-    },
-    {
-      "id": "dataStreamPublisherConfig",
-      "type": "pickString",
-      "description": "Publisher Configuration",
-      "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
-      ],
-      "default": "MarketDataPublisher",
-    },
-    {
-      "id": "pubSubServerConfig",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
-      ],
-      "default": "PubSubServer",
-    },
-    {
-      "id": "sparkConfigQuery",
-      "type": "promptString",
-      "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkdConfigQuery",
-      "type": "promptString",
-      "description": "Config Type(s)"
-    }
-  ]
-}

CMakeLists.txt

@@ -14,17 +14,12 @@ set(
   CACHE
   STRING
   "External dependencies root folder (should contain folders: lib, bin, include")
-
-# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
-# can run simultaneously without fighting over the same shm files (e.g., sparkd,
-# api server, etc.).
-set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
 set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
 set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
-set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
 if (WIN32)
   set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
   set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -36,13 +31,13 @@ endif()
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})
 
-add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
-if(DEFINED APP_DATA_DIR_OVERRIDE)
-  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
+if(BUILD_API_SERVER)
+  add_compile_definitions(SPARK_ENABLE_API_SERVER)
 endif()
-add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
+
 if(WIN32)
+  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
+  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
   # subdirs
   add_subdirectory(src/lib)
 else()
@@ -52,7 +47,13 @@ else()
   set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
   set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
 
-  if(BUILD_SPARKD)
+  if(BUILD_API_SERVER)
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
+    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
+  elseif(BUILD_SPARKD)
+    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
+
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
     add_compile_definitions(SPARK_ENABLE_CCAPI)
@@ -132,8 +133,6 @@ else()
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
       COMMAND
         ln ARGS -sf
         ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0

CMakePresets.json

@@ -15,13 +15,11 @@
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build/${presetName}",
+      "binaryDir": "${sourceDir}/build",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -32,12 +30,7 @@
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "spark"
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
       }
     },
 
@@ -53,8 +46,6 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
-        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "ON"
@@ -71,8 +62,7 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "sparkd"
+        }
       }
     },
 
@@ -105,72 +95,98 @@
     {
       "name": "sparkd-debug-clang",
       "inherits": "sparkd-debug",
-      "displayName": "Sparkd (Debug | Clang)",
+      "displayName": "Debug (clang)",
       "description": "Debug build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
       }
     },
 
     {
       "name": "sparkd-debug-gcc",
       "inherits": "sparkd-debug",
-      "displayName": "Sparkd (Debug | Gcc)",
+      "displayName": "Debug (gcc)",
       "description": "Debug build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
       }
     },
 
     {
       "name": "sparkd-release-clang",
       "inherits": "sparkd-release",
-      "displayName": "Sparkd (Release | Clang)",
+      "displayName": "Release (clang)",
       "description": "Release build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
       }
     },
 
     {
       "name": "sparkd-release-gcc",
       "inherits": "sparkd-release",
-      "displayName": "Sparkd (Release | Gcc)",
+      "displayName": "Release (gcc)",
       "description": "Release build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
+      }
+    },
+
+    {
+      "name": "spark-api-server-base",
+      "hidden": true,
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
       }
     },
 
     {
       "name": "spark-api-server-windows",
       "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)",
+      "displayName": "Spark Api Server",
       "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
-        "SHARED_MEMORY_PREFIX": "spark_api_server"
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        }
       }
     },
 
     {
       "name": "spark-api-server-linux",
       "inherits": "sparkd",
-      "displayName": "Spark Api Server (Linux)",
+      "displayName": "Spark Api Server",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
         },
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
         "BUILD_SPARKD": {
           "type": "BOOL",
           "value": "OFF"
@@ -181,11 +197,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
       }
     },
 
@@ -208,62 +220,13 @@
     },
 
     {
-      "name": "data-stream-suite-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
+      "name": "data-stream-suite",
       "inherits": "spark",
-      "displayName": "Data Stream Suite (Windows)",
+      "displayName": "Data Stream Suite",
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "SHARED_MEMORY_PREFIX": "data_stream"
-      }
-    },
-
-    {
-      "name": "data-stream-suite-linux",
-      "displayName": "Data Stream Suite (Linux)",
-      "description": "Data Stream utils ported over from Edge",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
-      "inherits": "sparkd",
-      "generator": "Ninja Multi-Config",
-      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "BUILD_ADMIN_WEB_APP": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SDK": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SPARKD": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_C_COMPILER": "clang",
-        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
       }
     }
   ],
@@ -281,7 +244,7 @@
 
     {
       "name": "spark-debug",
-      "displayName": "Spark (Debug)",
+      "displayName": "Debug",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Debug"
@@ -289,7 +252,7 @@
 
     {
       "name": "spark-release",
-      "displayName": "Spark (Release)",
+      "displayName": "Release",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Release"
@@ -317,28 +280,28 @@
     {
       "name": "sparkd-debug-clang",
       "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang)",
+      "displayName": "Debug (clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-debug-gcc",
       "configurePreset": "sparkd-debug-gcc",
-      "displayName": "Sparkd (Debug | Gcc)",
+      "displayName": "Debug (gcc)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-clang",
       "configurePreset": "sparkd-release-clang",
-      "displayName": "Sparkd (Release | Clang)",
+      "displayName": "Release (clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-gcc",
       "configurePreset": "sparkd-release-gcc",
-      "displayName": "Sparkd (Release | Gcc)",
+      "displayName": "Release (gcc)",
       "inherits": "sparkd"
     },
 
@@ -355,14 +318,14 @@
 
     {
       "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Windows | Debug)",
+      "displayName": "Spark Api Server (Debug)",
       "inherits": "spark-api-server-windows",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Windows | Release)",
+      "displayName": "Spark Api Server (Debug)",
       "inherits": "spark-api-server-windows",
       "configuration": "Release"
     },
@@ -381,14 +344,14 @@
     {
       "name": "spark-api-server-linux-debug",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Debug)",
+      "displayName": "Spark Api Server (Debug)",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-linux-release",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Release)",
+      "displayName": "Spark Api Server (Release)",
       "configuration": "Release"
     },
 
@@ -408,30 +371,16 @@
     },
 
     {
-      "name": "data-stream-suite-windows-debug",
-      "displayName": "Data Stream Suite (Windows | Debug)",
-      "configurePreset": "data-stream-suite-windows",
+      "name": "data-stream-suite-debug",
+      "displayName": "DataStreamSuite (Debug)",
+      "configurePreset": "data-stream-suite",
       "configuration": "Debug"
     },
 
     {
-      "name": "data-stream-suite-windows-release",
-      "displayName": "Data Stream Suite (Windows | Release)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "data-stream-suite-linux-debug",
-      "displayName": "Data Stream Suite (Linux | Debug)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-linux-release",
-      "displayName": "Data Stream Suite (Linux | Release)",
-      "configurePreset": "data-stream-suite-linux",
+      "name": "data-stream-suite-release",
+      "displayName": "DataStreamSuite (Release)",
+      "configurePreset": "data-stream-suite",
       "configuration": "Release"
     }
   ]

docker/sparkd/dev/Dockerfile

@@ -48,15 +48,9 @@ RUN apt-get update && apt-get install -y \
 # cmake
 RUN \
     curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-    | gpg --dearmor - \
-    | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-    | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+    | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
+    && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
     && apt-get update \
-    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
-    && apt-get install -y kitware-archive-keyring \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
-    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
 # node

docker/sparkd/dev/devcontainer.json

@@ -1,7 +1,6 @@
 {
   "image": "bluetradingsystems/sparkd-dev:latest",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
+  "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
   "hostRequirements": {
     "memory": "4gb",
     "storage": "6gb"
@@ -28,6 +27,6 @@
     "8003:8003"
   ],
   "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+    "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
   ]
 }

docker/sparkd/dev/devcontainer_initialize.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"

notes

@@ -0,0 +1,6 @@
+cd ~/spark/build/clang/debug/; cmake --build . --target install; rm -rf ~/spark/install/Debug/bin.zip; tar -zcvf  ~/spark/install/Debug/bin.zip ~/spark/install/Debug/bin; scp ~/spark/install/Debug/bin.zip bts@10.1.10.31:~/13000/; cd ~/spark/
+cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; docker container stop spark_13000; docker container start spark_13000
+cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; cp -r sparkbin ~/.sparkd-dev/mounts/sparkd_build/; 
+ES SECURITYTYPE_FUTURE at VENUE_GLBX
+./sparkd md dump --symbol ES --type future --venue glbx --platform mdp3onixsex
+BT5u53r

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BaseUtilities/Crono.cs

@@ -1,22 +1,41 @@
-using System.Diagnostics;
+using mbts.spark;
 
-namespace BCG.Utilities.General;
+using System;
+using System.Runtime;
 
-/// <summary>
-/// Summary description for Crono.
-/// </summary>
-public class Crono {
-   readonly Stopwatch myTimer = new();
+namespace BCG.Utilities.General
+{
+	/// <summary>
+	/// Summary description for Crono.
+	/// </summary>
+	public class Crono
+	{
+      public Crono()
+      {
+         myTimer = new HighPerformanceTimer();
+      }
 
-   public void start() =>
-      myTimer.Start();
+      public void start() 
+      {myTimer.start();}
 
-   public void stop() =>
-      myTimer.Stop();
+      public void stop()
+      {myTimer.stop();}
 
-   public TimeSpan time =>
-      myTimer.Elapsed;
+      public TimeSpan time
+      {
+         get 
+         {
+            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
+         }
+      }
+
+      public double miliseconds
+      {
+         get
+         {return myTimer.totalMilliseconds;}
+      }
+
+      private HighPerformanceTimer myTimer;
+	}
 
-   public double miliseconds =>
-      myTimer.Elapsed.TotalMilliseconds;
 }

src/DataStreamSuite/BlueRegistry/BlueRegistry.cs

@@ -1,192 +1,192 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Reflection;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public enum RegistryRoot { HKCU, HKLM }
-
-   [Serializable]
-   public class BlueRegistry
-   {
-      public const string BlueSubKey = @"Software\Blue Capital Group\";
-      public static readonly string ApplicationPrefix;
-      protected string myFullPath;
-
-      public BlueRegistry(string subPath)
-      {
-         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
-      }
-
-      public BlueRegistry(BlueRegistry parent, string subPath)
-      {
-         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
-      }
-
-      static BlueRegistry()
-      {
-         string app = "Applications";
-
-         Assembly ass = Assembly.GetEntryAssembly();
-         if (ass != null)
-         {
-            app = ass.GetName().Name;
-
-            object[] attrs = ass.GetCustomAttributes(
-               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
-
-            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
-            {
-               app = ((AssemblyTitleAttribute)attrs[0]).Title;
-            }
-         }
-
-         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
-      }
-
-      public object getValue(string name)
-      {
-         return getValue(RegistryRoot.HKCU, name);
-      }
-
-      public object getValue(string name, object defaultValue)
-      {
-         return getValue(RegistryRoot.HKCU, name, defaultValue);
-      }
-
-      public void setValue(string name, object value)
-      {
-         setValue(RegistryRoot.HKCU, name, value);
-      }
-
-      public void deleteValue(string name)
-      {
-         deleteValue(RegistryRoot.HKCU, name);
-      }
-
-      private RegistryKey myGetRegistryKey(RegistryRoot root)
-      {
-         if (root == RegistryRoot.HKCU)
-            return Registry.CurrentUser.CreateSubKey(myFullPath);
-         else
-            return Registry.LocalMachine.CreateSubKey(myFullPath);
-      }
-
-      #region Generic Methods
-      public virtual void setValue(RegistryRoot root, string name, object value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.SetValue(name, value);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name);
-         }
-      }
-
-      internal virtual void deleteValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.DeleteValue(name);
-         }
-      }
-
-      public virtual object getValue(RegistryRoot root, string name, object defaultValue)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name, defaultValue);
-         }
-      }
-      #endregion
-
-      #region Double
-      internal virtual void setDouble(RegistryRoot root, string name, double value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            DoubleRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual double getDouble(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return DoubleRegistryHandler.getValue(rk, name);
-         }
-      }
-      #endregion
-
-      #region Enum
-      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            EnumRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return EnumRegistryHandler.getValue(rk, name, enumType);
-         }
-      }
-      #endregion
-
-      #region Array
-      internal virtual void setArray(RegistryRoot root, string name, Array array)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            ArrayRegistryHandler.setValue(rk, name, array);
-         }
-      }
-
-      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return ArrayRegistryHandler.getValue(rk, name, elementType);
-         }
-      }
-      #endregion
-
-      #region SortedList
-      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.setValue(rk, name, slist);
-         }
-      }
-
-      internal virtual SortedList getSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return SortedListRegistryHandler.getValue(rk, name);
-         }
-      }
-
-      internal virtual void deleteSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.deleteValue(rk, name);
-         }
-      }
-      #endregion
-   }
-}
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      internal virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,9 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>
 
 </Project>

src/DataStreamSuite/BlueRegistry/PersistWindowState.cs

@@ -1,166 +1,166 @@
-using System;
-using System.Reflection;
-using System.Windows.Forms;
-using System.Drawing;
-using System.Diagnostics;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public class PersistWindowState
-   {
-      // event info that allows form to persist extra window state data
-      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
-      public event WindowStateHandler LoadState;
-      public event WindowStateHandler SaveState;
-
-      private Form myParent;
-      private BlueRegistry myRegistry;
-
-      private int myNormalLeft;
-      private int myNormalTop;
-      private int myNormalWidth;
-      private int myNormalHeight;
-      private bool myResizable = true;
-      private FormWindowState myWindowState;
-      private bool myAllowSaveMinimized = false;
-      private bool mySavePosition = true;
-      private bool mySaveState = true;
-
-      public PersistWindowState (bool savePosition)
-      {
-         mySavePosition = savePosition;
-      }
-
-      public PersistWindowState ()
-      {
-      }
-
-      public bool saveState
-      {
-         [DebuggerStepThrough]
-         set { mySaveState = value; }
-      }
-
-      public Form parent
-      {
-         [DebuggerStepThrough]
-         get { return myParent; }
-         set
-         {
-            myParent = value;
-            myRegistry = new BlueRegistry (myParent.Text);
-
-            // subscribe to parent form's events
-            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
-            myParent.Resize += new System.EventHandler(myOnResize);
-            myParent.Move += new System.EventHandler(myOnMove);
-            myParent.Load += new System.EventHandler(myOnLoad);
-
-            // get initial width and height in case form is never resized
-            myNormalWidth = myParent.Width;
-            myNormalHeight = myParent.Height;
-         }
-      }
-
-      public BlueRegistry blueRegistry
-      {
-         [DebuggerStepThrough]
-         get { return myRegistry; }
-         [DebuggerStepThrough]
-         set { myRegistry = value; }
-      }
-
-      public bool resizable
-      {
-         [DebuggerStepThrough]
-         set { myResizable = value; }
-      }
-
-      public bool allowSaveMinimized
-      {
-         [DebuggerStepThrough]
-         set { myAllowSaveMinimized = value; }
-      }
-
-      private void myOnResize(object sender, System.EventArgs e)
-      {
-         // save width and height
-         if (myParent.WindowState == FormWindowState.Normal && myResizable)
-         {
-            myNormalWidth = myParent.Width;
-            myNormalHeight = myParent.Height;
-         }
-      }
-
-      private void myOnMove(object sender, System.EventArgs e)
-      {
-         // save position
-         if(myParent.WindowState == FormWindowState.Normal)
-         {
-            myNormalLeft = myParent.Left;
-            myNormalTop = myParent.Top;
-         }
-         // save state
-         myWindowState = myParent.WindowState;
-      }
-
-      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
-      {
-         // save position, size and state
-         if (mySavePosition)
-         {
-            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
-            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
-         }
-
-         if (myResizable)
-         {
-            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
-            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
-         }
-
-         // check if we are allowed to save the state as minimized (not normally)
-         if (!myAllowSaveMinimized)
-         {
-            if(myWindowState == FormWindowState.Minimized)
-               myWindowState = FormWindowState.Normal;
-         }
-
-         if (mySaveState)
-            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
-        
-         // fire SaveState event
-         if (SaveState != null)
-            SaveState (this, myRegistry);
-      }
-
-      private void myOnLoad (object sender, System.EventArgs e)
-      {
-         if (mySavePosition)
-         {
-            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
-            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
-            myParent.Location = new Point (left, top);
-         }
-
-         if (myResizable)
-         {
-            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
-            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
-            myParent.Size = new Size (width, height);
-         }
-            
-         if (mySaveState)
-         {
-            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
-               (int) myParent.WindowState);
-            myParent.WindowState = windowState;
-         }
-
-         // fire LoadState event
-         if(LoadState != null)
-            LoadState (this, myRegistry);
-      }
-   }
-}
+using System;
+using System.Reflection;
+using System.Windows.Forms;
+using System.Drawing;
+using System.Diagnostics;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public class PersistWindowState
+   {
+      // event info that allows form to persist extra window state data
+      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
+      public event WindowStateHandler LoadState;
+      public event WindowStateHandler SaveState;
+
+      private Form myParent;
+      private BlueRegistry myRegistry;
+
+      private int myNormalLeft;
+      private int myNormalTop;
+      private int myNormalWidth;
+      private int myNormalHeight;
+      private bool myResizable = true;
+      private FormWindowState myWindowState;
+      private bool myAllowSaveMinimized = false;
+      private bool mySavePosition = true;
+      private bool mySaveState = true;
+
+      public PersistWindowState (bool savePosition)
+      {
+         mySavePosition = savePosition;
+      }
+
+      public PersistWindowState ()
+      {
+      }
+
+      public bool saveState
+      {
+         [DebuggerStepThrough]
+         set { mySaveState = value; }
+      }
+
+      public Form parent
+      {
+         [DebuggerStepThrough]
+         get { return myParent; }
+         set
+         {
+            myParent = value;
+            myRegistry = new BlueRegistry (myParent.Text);
+
+            // subscribe to parent form's events
+            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
+            myParent.Resize += new System.EventHandler(myOnResize);
+            myParent.Move += new System.EventHandler(myOnMove);
+            myParent.Load += new System.EventHandler(myOnLoad);
+
+            // get initial width and height in case form is never resized
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      public BlueRegistry blueRegistry
+      {
+         [DebuggerStepThrough]
+         get { return myRegistry; }
+         [DebuggerStepThrough]
+         set { myRegistry = value; }
+      }
+
+      public bool resizable
+      {
+         [DebuggerStepThrough]
+         set { myResizable = value; }
+      }
+
+      public bool allowSaveMinimized
+      {
+         [DebuggerStepThrough]
+         set { myAllowSaveMinimized = value; }
+      }
+
+      private void myOnResize(object sender, System.EventArgs e)
+      {
+         // save width and height
+         if (myParent.WindowState == FormWindowState.Normal && myResizable)
+         {
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      private void myOnMove(object sender, System.EventArgs e)
+      {
+         // save position
+         if(myParent.WindowState == FormWindowState.Normal)
+         {
+            myNormalLeft = myParent.Left;
+            myNormalTop = myParent.Top;
+         }
+         // save state
+         myWindowState = myParent.WindowState;
+      }
+
+      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
+      {
+         // save position, size and state
+         if (mySavePosition)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
+            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
+         }
+
+         if (myResizable)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
+            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
+         }
+
+         // check if we are allowed to save the state as minimized (not normally)
+         if (!myAllowSaveMinimized)
+         {
+            if(myWindowState == FormWindowState.Minimized)
+               myWindowState = FormWindowState.Normal;
+         }
+
+         if (mySaveState)
+            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
+        
+         // fire SaveState event
+         if (SaveState != null)
+            SaveState (this, myRegistry);
+      }
+
+      private void myOnLoad (object sender, System.EventArgs e)
+      {
+         if (mySavePosition)
+         {
+            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
+            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
+            myParent.Location = new Point (left, top);
+         }
+
+         if (myResizable)
+         {
+            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
+            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
+            myParent.Size = new Size (width, height);
+         }
+            
+         if (mySaveState)
+         {
+            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
+               (int) myParent.WindowState);
+            myParent.WindowState = windowState;
+         }
+
+         // fire LoadState event
+         if(LoadState != null)
+            LoadState (this, myRegistry);
+      }
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnExpression">
-    <Scanner Include="ColumnExpressionTokens.lex" />
-    <Parser Include="ColumnExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -1,62 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-public class ColumnExpression {
-   public ColumnExpression(string name, Type? type = null, object? value = null) {
-      Name = name;
-      Type = type
-         ?? value?.GetType()
-         ?? throw new ArgumentException(
-            "Type must be specified explicitly when value is null", nameof(type));
-      Value = value
-         ?? DBNull.Value;
-   }
-
-   public string Name { get; }
-
-   public Type Type { get; }
-
-   public object Value { get; }
-
-   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
-      // Try to convert value to 'type' to verify they are compatible
-      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
-      // account for implicit conversions (e.g., int -> long), just inheritance.
-      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
-         throw new ArgumentException(
-            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
-      }
-      return new(name, type ?? typeof(T), value);
-   }
-
-   internal static Type ParseType(string typeId) {
-      return Candidates().ExceptNull().FirstOrDefault() ??
-         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
-
-      IEnumerable<Type?> Candidates() {
-         if (typeId.Length == 0) {
-            yield break;
-         }
-         yield return Type.GetType(typeId);
-         yield return Type.GetType($"System.{typeId}");
-         if (typeId.Length > 1) {
-            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
-         }
-         yield return typeId.ToLower() switch {
-            "float64" or "double" or "real64" or "number" => typeof(double),
-            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
-            "int16" or "short" or "integer16" or "signed16" => typeof(short),
-            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
-            "int64" or "long" or "integer64" or "signed64" => typeof(long),
-            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
-            "uint32" or "uint" or "unsigned" => typeof(uint),
-            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
-            "char" => typeof(char),
-            "byte" or "uint8" => typeof(byte),
-            "bool" or "boolean" => typeof(bool),
-            "string" or "text" => typeof(string),
-            "date" or "datetime" => typeof(DateTime),
-            _ => null
-         };
-      }
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -1,49 +0,0 @@
-%namespace Bts.ColumnExpressions
-%partial
-%visibility internal
-%tokentype ColumnExpressionToken
-%parsertype ColumnExpressionParser
-
-%union {
-   public int Integer;
-   public double Real;
-   public bool Boolean;
-   public string String;
-   public string Identifier;
-   public string TypeId;
-   public DateTime DateTime;
-   public Tuple<string, Type> ColumnDefinition;
-   public ColumnExpression ColumnExpression;
-   public ImmutableList<ColumnExpression> ColumnExpressions;
-}
-
-%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
-%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
-
-%nonassoc EQUALS
-
-%type <ColumnDefinition> ColumnDefinition
-%type <ColumnExpression> ColumnExpression
-%type <ColumnExpressions> ColumnExpressions
-
-%%
-
-ColumnExpressions :
-   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
-   ;
-
-ColumnExpression :
-   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
-   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
-   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
-   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
-   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
-   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
-   ;
-
-ColumnDefinition :
-   IDENTIFIER { $$ = new($1.Identifier, null); }
-   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
-   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
-   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnExpressionInterpreter {
-   public static IList<ColumnExpression> Evaluate(Stream stream) {
-      var parser = new ColumnExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
-         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.Text;
-
-partial class ColumnExpressionParser {
-   public ColumnExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-partial class ColumnExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -1,85 +0,0 @@
-%namespace Bts.ColumnExpressions
-%scannertype ColumnExpressionScanner
-%visibility internal
-%tokentype ColumnExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-BooleanConstant          true|false
-StringConstant           '([^\'\\]*|\\.)*'
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
-IntegerConstant          -?[0-9]+
-NullConstant             'null'
-RealConstant             -?[0-9]+\.[0-9]+
-
-/* Relational Operators */
-Equals                   =
-OfType                   /
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-NamespacePrefix          {Identifier}\.
-TypeId                   {NamespacePrefix}*{Identifier}
-
-/* Utility Tokens */
-OpenParenthesis          \(
-CloseParenthesis         \)
-Comma                    ,
-QuoteOpen                \[
-QuoteClose               \]
-
-/* Comments */
-LineComment \/\/[^\n\r]*(\n|\r|\r\n)
-
-%%
-
-{Equals} {
-   return (int)ColumnExpressionToken.EQUALS;
-}
-
-{OfType} {
-   return (int)ColumnExpressionToken.OF_TYPE;
-}
-
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
-   return (int)ColumnExpressionToken.STRING_CONSTANT;
-}
-
-{BooleanConstant} {
-   yylval.Boolean = yytext.ToLower() switch {
-      "true" => true,
-      "false" => false,
-      _ => throw new Exception(
-         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
-   };
-   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
-}
-
-{DateTimeConstant} {
-   yylval.DateTime = DateTime.Parse(yytext);
-   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
-}
-
-{IntegerConstant} {
-   yylval.Integer = int.Parse(yytext);
-   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
-}
-
-{RealConstant} {
-   yylval.Real = double.Parse(yytext);
-   return (int)ColumnExpressionToken.REAL_CONSTANT;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnExpressionToken.IDENTIFIER;
-}
-
-{TypeId} {
-   yylval.TypeId = yytext;   
-   return (int)ColumnExpressionToken.TYPE_ID;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnSortExpression">
-    <Scanner Include="ColumnSortExpressionTokens.lex" />
-    <Parser Include="ColumnSortExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -1,11 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-public class ColumnSortExpression {
-   public ColumnSortExpression(string name, bool isAscending = true) {
-      Name = name;
-      IsAscending = isAscending;
-   }
-
-   public string Name { get; }
-   public bool IsAscending { get; }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -1,29 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%partial
-%visibility internal
-%tokentype ColumnSortExpressionToken
-%parsertype ColumnSortExpressionParser
-
-%union {
-   public string Identifier;
-   public ColumnSortExpression Column;
-   public ImmutableList<ColumnSortExpression> Columns;
-}
-
-%token ASCENDING, DESCENDING, IDENTIFIER
-
-%type <Column> Column
-%type <Columns> Columns
-
-%%
-
-Columns :
-   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | Columns Column { $$ = $1.Add($2); }
-   ;
-
-Column :
-   IDENTIFIER { $$ = new($1.Identifier, true); }
-   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
-   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
-   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnSortExpressionInterpreter {
-   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
-      var parser = new ColumnSortExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
-         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnSortExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.Text;
-
-partial class ColumnSortExpressionParser {
-   public ColumnSortExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnSortExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-partial class ColumnSortExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -1,30 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%scannertype ColumnSortExpressionScanner
-%visibility internal
-%tokentype ColumnSortExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-Ascending                +
-Descending               -
-
-%%
-
-{Ascending} {
-   return (int)ColumnSortExpressionToken.ASCENDING;
-}
-
-{Descending} {
-   return (int)ColumnSortExpressionToken.DESCENDING;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnSortExpressionToken.IDENTIFIER;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -9,17 +9,9 @@
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
-    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-  </ItemGroup>
-
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -1,26 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Reactive.Linq;
-
-public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
-      source as ViewStream ?? new ObservableViewStream(source);
-
-   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<DataPacket> observer) {
-         source.onChange += OnChange;
-         return Disposable.Create(() => source.onChange -= OnChange);
-
-         void OnChange(object? newValue) =>
-            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
-               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
-      }
-   }
-}

src/DataStreamSuite/DataStream/Misc/PandasScriptStream.cs

@@ -1,29 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-
-using System;
-using System.IO;
-
-public class PandasScriptStream : ViewStream {
-   public PandasScriptStream(ViewStream vs, string file)
-      : this(vs, file, null) { }
-
-   public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
-      : this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
-
-   public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
-      mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
-
-      string readFile() {
-         try {
-            return File.ReadAllText(file);
-         } catch (Exception ex) {
-            Log.Gen.error($"PandasScriptStream error reading file: {ex}");
-            throw;
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,130 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-using BCG.Extensions;
-using BCG.Utilities.General;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Runtime.CompilerServices;
-using System.Text;
-
-public class PandasStream : ViewStream, IWorker {
-   readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
-   readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
-   readonly string pythonExe;
-   readonly ViewStreamDatabase vsd;
-   readonly CSVSaverStream inputCsvStream;
-   readonly Lazy<string> pythonCode;
-   IDictionary<int, IBlueRow>? lastRowMap = null;
-   int inputVersion = 0;
-   int outputVersion = 0;
-   DateTime lastFireTime = DateTime.MinValue;
-   TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
-   IDisposable? dispose;
-
-   public PandasStream(ViewStream vs, string dataFrameExpression)
-      : this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
-
-   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
-      pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
-      this.pythonExe = pythonExe;
-      this.vsd = vsd;
-      // This should fire an update when it writes to the output file
-      inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
-      pythonCode = new(getPythonCode);
-
-      string getPythonCode() {
-         var imports =
-@"import numpy as np
-import pandas as pd
-import sys
-";
-         var bd = new StringBuilder(imports);
-         bd.AppendLine($"df=pd.read_csv('{inputFile}')");
-         bd.AppendLine(dataFrameExpression);
-         bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
-         return bd.ToString();
-      }
-   }
-
-   protected override void myOnFirstSubscriber() {
-      vsd.addWorker(this);
-      inputCsvStream.onChange += onInputChanged;
-      this.dispose = Disposable.Create(dispose);
-
-      void onInputChanged(object newValue) =>
-         inputVersion++;
-
-      void dispose() {
-         inputCsvStream.onChange -= onInputChanged;
-         vsd.removeWorker(this);
-         deleteFile(inputFile);
-         deleteFile(outputFile);
-      }
-
-      void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
-         if (!File.Exists(file)) {
-            return;
-         }
-         try {
-            File.Delete(file);
-         } catch (Exception ex) {
-            Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose(ref dispose);
-
-   static void executePythonCode(string pythonCode, string? pythonExe = null) {
-      pythonExe ??= "python";
-      var start = new ProcessStartInfo {
-         FileName = pythonExe,
-         Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
-         UseShellExecute = false,
-         RedirectStandardOutput = true
-      };
-      using var process = Process.Start(start)
-         ?? throw new InvalidOperationException("Failed to launch python");
-      process.WaitForExit();
-   }
-
-   void IWorker.doWork() {
-      if (outputVersion == inputVersion) {
-         return;
-      }
-      var now = DateTime.Now;
-      var elapsedSinceLastFire = now - lastFireTime;
-      if (elapsedSinceLastFire < fireThrottle) {
-         return;
-      }
-      outputVersion = inputVersion;
-      executePythonCode(pythonCode.Value, pythonExe);
-      var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
-      var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
-      var comp = BlueRowEqualityComparer.Instance;
-      var deletedRows = new BlueTable(schema);
-      var addedRows = new BlueTable(schema);
-      if (lastRowMap is not null) {
-         deletedRows.SetItems(lastRowMap.Where(oldRow =>
-            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
-         addedRows.SetItems(rowMap.Where(newRow =>
-            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
-      }
-      lastRowMap = rowMap;
-      lastFireTime = DateTime.Now;
-      myFireEvent(new DataPacket(rows, addedRows, deletedRows));
-   }
-}

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -1,79 +0,0 @@
-using BCG.Utilities.General;
-
-using System;
-
-namespace BCG {
-    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
-        class State {
-            public int n_sends = 0;
-            public int n_recvs = 0;
-        }
-        private readonly string _ip;
-        private readonly int _port;
-        private readonly ViewStreamDatabase _vsd;
-        State _state = new State();
-        private readonly int _msg_size;
-
-        private IBlueTableSchema _schema;
-        private TimeSpan _send_period;
-        private ITCPConnection _connection = null;
-        private byte[] _buff;
-
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
-        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
-            _ip = ip;
-            _port = port;
-            _msg_size = msg_size;
-            _vsd = vsd;
-            _schema = BlueDataUtility.form_schema("TCPClient",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("n_sends", TypeCode.UInt32),
-            Tuple.Create("n_recvs", TypeCode.UInt32));
-            _send_period = TimeSpan.FromSeconds(send_period_seconds);
-            _buff = new byte[msg_size];
-            for (int i = 0; i < _buff.Length; i++) {
-                _buff[i] = (byte)i;
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_rci = new RemoteConnectionInfo(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            if (_connection != null) {
-                _connection.send(_buff);
-                _state.n_sends++;
-            }
-            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
-            BlueTable rows = new BlueTable(r.schema);
-            rows.addRow(r);
-            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        public void onConnection(ITCPConnection connection) {
-            _connection = connection;
-            _connection.register(this);
-        }
-        public bool isDisposed { get; }
-
-        public void onData(RemoteMessage msg) {
-            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
-            _state.n_recvs++;
-        }
-
-        public void Dispose() {
-
-        }
-
-
-    }
-}

src/DataStreamSuite/DataStream/Misc/test.py

@@ -1,3 +0,0 @@
-file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
-with open(file_path, "w") as f:
-    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -1,37 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-
-using System;
-
-public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
-   readonly IObservable<DataPacket> source;
-   IDisposable? subscription;
-
-   public ObservableViewStream(IObservable<DataPacket> source) =>
-      this.source = source;
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      if (subscription is not null) {
-         subscription.Dispose();
-         subscription = null;
-      }
-   }
-
-   protected override void myOnFirstSubscriber() {
-      Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
-   }
-
-   protected override void myOnLastSubscriber() {
-      Log.notnull(subscription);
-      subscription.Dispose();
-      subscription = null;
-   }
-
-   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
-      source.Subscribe(observer);
-}

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -0,0 +1,32 @@
+using BCG;
+
+namespace BCG
+{
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("add_to_cloud")]
+   public class AddToCloudStream : ViewStream
+   {
+      private readonly string _name, _stream_exp;
+      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
+      public AddToCloudStream(string name, params string[] lines)
+      : this(name, 0, lines)
+      {
+      }
+
+      public AddToCloudStream(string name, int persist, params string[] lines)
+      {
+         _name = name;
+         _stream_exp = getExpression(lines);
+         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
+         mySetRealSource(new TableStream("cloud_streams"));
+      }
+
+      static string getExpression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,22 +1,31 @@
+using BCG;
+
 using System;
 using System.Collections.Generic;
 
+using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
+using System.Collections.Generic;
 using BCG.Utilities.General;
-using bts;
 
-namespace BCG {
+namespace BCG
+{
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IWorker {
-      public class StreamPubUnit {
+   public class CloudStreamManager : IWorker
+   {
+      public class StreamPubUnit
+      {
          public string name;
          public string stream_expression;
          public ViewStream stream;
          public bool persist;
          public ViewPublisher view_publisher;
 
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
+         {
             name = name_;
             stream_expression = stream_expression_;
             stream = stream_;
@@ -24,7 +33,7 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream? build_stream(string stream_expression)
+      static ViewStream build_stream(string stream_expression)
       {
          try
          {
@@ -32,7 +41,9 @@ namespace BCG {
             var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
@@ -41,37 +52,34 @@ namespace BCG {
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       private ViewStream _stream = new TableStream("cloud_streams");
       private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
-
-      public RemoteQueryManager() {
+      public CloudStreamManager()
+      {
          _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
+
       }
       private void _on_str(object obj)
       {
-         if (obj is not DataPacket dp) {
-            return;
-         }
+         DataPacket dp = obj as DataPacket;
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
             bool persist = r.get<bool>("persist");
             string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name)) {
-               _spawn_remote_stream(name, stream_exp, persist);
+            if (!_stream_pub_map.ContainsKey(name))
+            {
+               _assign(name, stream_exp, persist);
             }
          }
       }
 
-      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
+      private void _assign(string name, string stream_expression, bool persist)
+      {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
-         if (persist) {
+         if (persist)
+         {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -79,12 +87,14 @@ namespace BCG {
          _stream_pub_map.Add(name, u);
       }
 
-      private void _expose_to_ws(string name, ViewStream stream) {
+      private void _expose_to_ws(string name, ViewStream stream)
+      {
          // TODO: john
       }
 
-      public void doWork() {
+      public void doWork()
+      {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -1,48 +0,0 @@
-using BCG;
-
-using bts;
-
-using System;
-
-namespace BCG {
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("remoteQuery")]
-   public class RemoteQueryStream : ViewStream {
-      private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = new PublisherCore();
-      private ViewStream _source_str;
-
-      public RemoteQueryStream(params string[] lines) {
-         _stream_exp = get_expression(lines);
-         _name = Guid.NewGuid().ToString();
-         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
-         //_name.Replace("-", "_");
-
-      }
-
-      protected override void myOnFirstSubscriber() {
-         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
-         byte[] rem_req_bytes = rem_req.serialize();
-         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
-         _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
-      }
-
-      protected override void myOnLastSubscriber() {
-         //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
-      }
-
-      static string get_expression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-
-      private void _handle_data(object o) {
-         myFireEvent(o);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -1,31 +0,0 @@
-namespace bts {
-   using System.IO;
-   using BCG;
-
-   using System.IO;
-   using System.Threading.Tasks.Dataflow;
-
-   public class RemoteRequest {
-      public readonly string stream_expression;
-      public readonly string published_table_name;
-      public RemoteRequest(string stream_expression_, string published_table_name_) {
-         stream_expression = stream_expression_;
-         published_table_name = published_table_name_;
-      }
-      public byte[] serialize() {
-         var str = new MemoryStream();
-         var bw = new BinaryWriter(str);
-         bw.write_string_cpp(stream_expression);
-         bw.write_string_cpp(published_table_name);
-         return str.ToArray();
-      }
-
-      public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
-         BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
-         string stream_expression_ = br.read_string_cpp();
-         string published_table_name_ = br.read_string_cpp();
-         return new RemoteRequest(stream_expression_, published_table_name_);
-
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -1,54 +1,59 @@
-// using BCG;
+using BCG;
 
-// using bts;
+using System.Linq;
 
-// using System.Linq;
+namespace BCG
+{
+   public class StreamOffloadRequestor
+   {
+      public class OffloadOp
+      {
+         string name;
+         string stream_exp;
+      }
+      private IBlueTableSchema _schema;
+      private IBlueRowCollection _rows;
+      private PubTableBroker _broker = new PubTableBroker();
+      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+      public StreamOffloadRequestor()
+      {
+         _schema = new BlueTableSchema("cloud_streams",
+                                   new ColumnInfo("id", 0, typeof(int)),
+                                   new ColumnInfo("name", 1, typeof(string)),
+                                   new ColumnInfo("persist", 2, typeof(bool)),
+                                   new ColumnInfo("stream_exp", 3, typeof(string)));
+         _existing_offloads_vs.onChange += _on_str;
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         _rows = dp.newRows;
+         // this is needed so the broker knows the state of the table
+         _broker.createTable(_rows);
+      }
+      public void add_to_cloud(string name, string stream_exp, bool persist)
+      {
+         // check if it exists
+         if (_rows == null)
+         {
+            BlueTable table = new BlueTable(_schema);
+            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+            table.addRow(nr);
+            _broker.createTable(table);
+            // create table  
+         }
+         else
+         {
+            if (exists(name))
+               return;
+            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+            _broker.addRow(nr);
+         }
+      }
 
-// namespace BCG {
-//    // used on the client
-//    public class StreamOffloadRequestor {
-//       public class OffloadOp {
-//          string name;
-//          string stream_exp;
-//       }
-//       private IBlueTableSchema _schema;
-//       private IBlueRowCollection _rows;
-//       private PubTableBroker _broker = new PubTableBroker();
-//       ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-//       public StreamOffloadRequestor() {
-//          _schema = new BlueTableSchema("cloud_streams",
-//                                    new ColumnInfo("id", 0, typeof(int)),
-//                                    new ColumnInfo("name", 1, typeof(string)),
-//                                    new ColumnInfo("persist", 2, typeof(bool)),
-//                                    new ColumnInfo("stream_exp", 3, typeof(string)));
-//          _existing_offloads_vs.onChange += _on_str;
-//       }
-//       private void _on_str(object obj) {
-//          DataPacket dp = obj as DataPacket;
-//          _rows = dp.newRows;
-//          // this is needed so the broker knows the state of the table
-//          _broker.create_table(_rows);
-//       }
-//       public void add_to_cloud(string name, string stream_exp, bool persist) {
-//          // check if it exists
-//          if (_rows == null) {
-//             BlueTable table = new BlueTable(_schema);
-//             IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-//             table.addRow(nr);
-//             _broker.create_table(table);
-//             // create table  
-//          } else {
-//             if (exists(name))
-//                return;
-//             IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-//             // TODO: use make_batch_changes instead
-//             Log.assert(false, "Fix addRow");
-//             //_broker.addRow(nr);
-//          }
-//       }
-
-//       bool exists(string name) {
-//          return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-//       }
-//    }
-// }
+      bool exists(string name)
+      {
+         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs

@@ -1,14 +1,17 @@
 using System;
 using System.Collections;
 
-namespace BCG {
-   internal class AddColumnsRow : IBlueRow {
+namespace BCG
+{
+   internal class AddColumnsRow : IBlueRow
+   {
       private IBlueTableSchema mySchema;
       private IBlueRow myRow;
       //private object[] myCachedValues;
 
       #region Constructors
-      public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
+      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
+      {
          mySchema = schema;
          myRow = row;
          //myCachedValues = new object[schema.columnNames.Length];
@@ -16,78 +19,85 @@ namespace BCG {
       #endregion
 
       #region IBlueRow members
-      public IBlueTableSchema schema { get { return mySchema; } }
+      public IBlueTableSchema schema{get{return mySchema;}}
       #endregion
 
       #region ICollection members
-      public int Count { get { return mySchema.columnNames.Length; } }
+      public int Count {get{return mySchema.columnNames.Length;}}
 
-      public bool IsSynchronized { get { return false; } }
+      public bool IsSynchronized {get{return false;}}
 
-      public object SyncRoot { get { return this; } }
+      public object SyncRoot {get{return this;}}
 
-      public void CopyTo(Array array, int index) {
+      public void CopyTo (Array array, int index)  
+      {
          int i = index;
          foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
+            array.SetValue (o, i++);
       }
       #endregion
 
       #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
+      public bool IsFixedSize {get{return true;}}
 
-      public bool IsReadOnly { get { return true; } }
+      public bool IsReadOnly {get{return true;}}
 
       /// <summary>
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public object this[object key] {
-         get {
+      public object this [object key]
+      {
+         get
+         {
             if (key is string)
                return this[(string)key];
             return this[(int)key];
          }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+         set {throw new NotSupportedException ("The IBlueRow is readonly");}
       }
 
-      public object this[int index] {
-         get {
+      public object this[int index]
+      {
+         get
+         {
             //if (myCachedValues[index] == null)
             //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
             //return myCachedValues[index];
-            return mySchema[index].isConstant ? DBNull.Value : myRow[index];
+            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
          }
       }
 
-      public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
+      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}
 
-      public ICollection Keys { get { return mySchema.columnNames; } }
+      public ICollection Keys{get{return mySchema.columnNames;}}
 
-      public ICollection Values {
-         get {
+      public ICollection Values
+      {
+         get
+         {
             object[] retVal = new object[mySchema.columnNames.Length];
-            CopyTo(retVal, 0);
+            CopyTo (retVal, 0);
             return retVal;
          }
       }
 
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
 
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
 
-      public bool Contains(object key) { return mySchema[(string)key] != null; }
+      public bool Contains (object key){return mySchema[(string)key] != null;}
 
       /// <summary>
       /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
       /// iterate through the values, typecast this into an IEnumerable
       /// </summary>
       /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
+      
+      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
 
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
       #endregion
 
    }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,34 +1,13 @@
 using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using BCG.Utilities.General;
+namespace BCG
+{
 
-namespace BCG {
-   public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_exp(string exp) {
-         string[] toks = exp.Split(':');
-         string col_name = toks[0];
-         string col_value = toks[1];
-         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return Tuple.Create(col_name, tup.Item1, tup.Item2);
-      }
-
-      private Tuple<string, Type, object> _new_col;
-      private IBlueTableSchema _out_schema;
-
-      public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
-         mySetRealSource(rs);
-      }
-
-      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null)
-            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
-         return _out_schema;
-      }
-
-      private IBlueRow _map(IBlueRow r) {
-         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         rv.setValue(_new_col.Item1, _new_col.Item3);
-         return rv;
-      }
+   public class AddColumnsStream : MapStream
+   {
+      public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef))  {}    
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -1,12 +0,0 @@
-namespace bts {
-    using BCG;
-
-    using System;
-    using System.Collections;
-
-    public class AddConstantColumnRow {
-        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
-
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,52 +1,73 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-
-using Bts.ColumnExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Provides methods to add columns to the schema
-/// </summary>
-/// <param name="columnsExpression">
-/// String represting the columns to add to the output table
-/// </param>
-public class ColumnAdder : IStreamMapper {
-   // Store tuples of (column, index) to preserve order
-   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
-   protected IBlueTableSchema? mySchema;
-
-   public ColumnAdder(string columnsExpression) =>
-      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
-         throw new Exception("Invalid columnn expression: " + columnsExpression))
-         .Select((c, i) => (c, i))
-         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
-
-   [MemberNotNull(nameof(mySchema))]
-   public virtual IBlueRow map(IBlueRow row) {
-      // Only rebuild schema if necessary (i.e., if it's null)
-      mySchema ??= buildSchema(row.schema, myNewColumns);
-      return new AddColumnsRow(mySchema, row);
-   }
-
-   [MemberNotNull(nameof(mySchema))]
-   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
-      // Always rebuild schema (even if it's already non-null)
-      mySchema = buildSchema(inputSchema, myNewColumns);
-
-   [MemberNotNull(nameof(mySchema))]
-   static BlueTableSchema buildSchema(
-      IBlueTableSchema schema,
-      IDictionary<string, (ColumnExpression column, int index)> newColumns
-   ) {
-      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
-      schema.columnInfo.CopyTo(cis, 0);
-      foreach (var (col, index) in newColumns.Values) {
-         var schemaIdx = schema.columnInfo.Length + index;
-         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
-      }
-      return new BlueTableSchema(cis, schema.tableName);
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.AddColumnsExpression;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Provides methods to add columns to the schema
+//    /// </summary>
+//    public class ColumnAdder : IStreamMapper
+//    {
+//       protected IDictionary myNewColumns;
+//       protected IBlueTableSchema mySchema;
+
+//       static syntax myParser =
+//          new syntax();
+
+//       /// <summary>
+//       /// string represting the columns to be added to the output table. 
+//       /// </summary>
+//       /// <param name="colAddExpression"></param>
+//       public ColumnAdder (string columnDef)
+//       {
+//          AddColumnsExpression ace;
+
+//          if (DataStreamLoadFileConfig.instance.useStaticParser)
+//          {
+//             lock (myParser)
+//             {
+//                ace =
+//                (AddColumnsExpression)myParser.Parse(columnDef);
+//             }
+//          }
+//          else
+//          {
+//             syntax parser = new syntax();
+//             ace =
+//             (AddColumnsExpression)parser.Parse(columnDef);
+//          }
+
+// 	 if (ace == null)
+// 	   throw new Exception ("Invalid columnn expression: " + columnDef);
+//          myNewColumns = ace.columns;
+//       }
+         
+//       public virtual IBlueRow map (IBlueRow row)
+//       {
+//          if (mySchema == null)
+//             myConstructSchema (row.schema);
+//          return new AddColumnsRow (mySchema, row);
+//       }
+
+//       public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
+//       {
+//          myConstructSchema (inputSchema);
+//          return mySchema;
+//       }
+
+//       private void myConstructSchema (IBlueTableSchema schema)
+//       {
+//          ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
+//          int ciIndex = 0;
+//          foreach (ColumnInfo ci in schema.columnInfo)
+//             cis[ciIndex++] = ci;
+//          foreach (DictionaryEntry de in myNewColumns)
+//             cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++, 
+//                ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
+//          mySchema = new BlueTableSchema (cis, schema.tableName);
+//       }
+
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs

@@ -1,24 +1,23 @@
-using bts;
-
 using System;
 using System.Collections;
 
-namespace BCG {
+namespace BCG
+{
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation("distinct")]
-	public class DistinctRowStream : ViewStream {
-		public DistinctRowStream(ViewStream stream, params string[] cols) {
-			var g_by_eval = new DistinctRowEvaluator(cols);
-			ViewStream r = new GroupByStream(stream, g_by_eval);
-			var proj_exp = String.Empty;
-			foreach (var c in cols)
-				proj_exp += $"{c} ";
-			proj_exp = proj_exp.Trim();
-			r = new NewProjectStream(r, proj_exp);
-			mySetRealSource(r);
+	[DataStreamUserRespresentation ("distinct")]
+	public class DistinctRowStream : ViewStream
+	{
+		public DistinctRowStream (ViewStream stream, params string[] cols)
+		{
+         ArrayList groupByCols = new ArrayList ();
+         foreach (string col in cols)
+            foreach (string c in col.Split (' '))
+               groupByCols.Add (c);
+         mySetRealSource (new GroupByStream (stream, 
+            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
 		}
 	}
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -0,0 +1,49 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
+
+// using System.Runtime.Serialization.Formatters.Binary;
+
+// namespace BCG
+// {
+//    public class DumpLoaderStream : ViewStream, ISideEffectStream
+//    {
+//       //wtf
+//       //public class Exception : System.Exception
+//       //{public Exception(string msg) : base(msg) { }}
+
+//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
+//       {
+//          mySetRealSource(
+//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+//                ViewStreamDatabase.instance));
+//       }
+
+//       public DumpLoaderStream(string fileName)
+//       {
+//         if (fileName == null)
+//            throw new LoaderStream.Exception("File name cannot be null");
+
+//         for (int i = 0; i < 5; i++)
+//            try
+//            {
+//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+//               {
+//                  IClientDeserializer deser = new ClientUniversalDeserializer();
+//                  byte[] the_ops;
+//                  Array the_rows;
+//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
+
+//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
+//               }
+//               break;
+//            }
+//            catch (IOException)
+//            {
+//               System.Threading.Thread.Sleep(50);
+//            }
+
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,25 +1,48 @@
-using BCG;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
 
-namespace bts {
-   // This is the other side of the SaverStream, its "loads" the stream and fires once
+using System.Runtime.Serialization.Formatters.Binary;
+
+namespace BCG
+{
    [DataStreamUserRespresentation("loader")]
-   public class LoaderStream : ViewStream {
+   public class LoaderStream : ViewStream, ISideEffectStream
+   {
+      public class Exception : System.Exception
+      {public Exception(string msg) : base(msg) { }}
 
-      private string _filename;
-      private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
-
-
-      public LoaderStream(string filename) {
-         _filename = filename;
+      public LoaderStream(string fileName, int secondsBetweenPolls)
+      {
+         mySetRealSource(
+            new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+               ViewStreamDatabase.instance));
       }
 
-      protected override void myOnFirstSubscriber() {
-         byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
-         StreamIOMessage msg = _deser.deserialize(ser_msg);
-         Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
-         IBlueRowCollection rows = msg.create_message.rows;
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
+      public LoaderStream(string fileName)
+      {
+        if (fileName == null)
+           throw new LoaderStream.Exception("File name cannot be null");
+
+        for (int i = 0; i < 5; i++)
+           try
+           {
+              using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+              {
+                  BinaryFormatter bf = new BinaryFormatter();
+#pragma warning disable SYSLIB0011
+                  BlueTable bt = (BlueTable)bf.Deserialize(fs);
+#pragma warning restore SYSLIB0011
+                  mySetRealSource(new BlueRowCollectionStaticStream(bt));
+              }
+              break;
+           }
+           catch (IOException)
+           {
+              System.Threading.Thread.Sleep(50);
+           }
+
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,11 +5,9 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using BCG;
 
-namespace bts {
-   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
-   // which just writes one update, as a create)
+namespace BCG {
+   // Writes all the udpates to a stream in a file.
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -17,16 +15,11 @@ namespace bts {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
-   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
-   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
-   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
-   // file is created. The file names are token_epoch_datetime
-   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;
       private FileStream _out_fs;
-      private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
+      private IBlueDataSerializer _serializer;
       private string _file_name = String.Empty;
       private long _rollover_size;
       private readonly string _directory_path;
@@ -95,19 +88,20 @@ namespace bts {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (first_write) {
-            // first file always starts with create
-            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
+         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
+         if (_write_create) {
+            // new files always starts with create
+            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
          } else
-            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
+            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
+
          BinaryWriter bw = new BinaryWriter(_out_fs);
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         //bw.Write(utc_epoch);
-         //bw.Write(serialized_update.Length);
+         bw.Write(utc_epoch);
+         bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -0,0 +1,190 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Runtime.Serialization.Formatters.Binary;
+using BCG.Utilities.General;
+using bts.collections;
+using System.Reflection.Metadata;
+
+
+namespace BCG {
+
+   public class PlaybackStream : ViewStream, IWorker {
+      private readonly PlaybackType _type;
+      private BlueTable _current_table;
+      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+      // these are the packets already downloaded from the current file
+      private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+      //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+      private IEnumerable<string> _file_names;
+      private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+      private DateTime _playback_t0, _playback_t1;
+      private bool _all_files_read = false;
+      SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
+
+      public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+         _type = PlaybackType.Streaming;
+         _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+         _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+         string token_directory_name = $"{directory_name}\\{token}";
+         bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+         List<string> files_in_range = new List<string>();
+         if (Directory.Exists(token_directory_name)) {
+            _file_names =
+         Directory.GetFiles(token_directory_name).Where(f => {
+            DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+            return file_time > _playback_t0 && file_time < _playback_t1;
+         });
+         }
+      }
+
+      protected override void myOnFirstSubscriber() {
+         FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+         _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+         if (snapshot_packet != null) {
+            foreach (PlaybackPacket p in _snapshot_data.packets) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+               for (int i = 0; i < ops.Length; i++) {
+                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+                  byte op = ops[i];
+                  if (op == StreamOpCode.create) {
+                     IBlueRowCollection full_table = rows_args[0];
+                     _current_table = new BlueTable(full_table);
+                  } else if (op == StreamOpCode.multipleRemoveAdd) {
+                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
+                  } else {
+                     bts.Log.assert(false, $"Oprator: {op} is not expected");
+                  }
+                  if (p.idx == snapshot_packet.idx) {
+                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+                     break;
+                  }
+               }
+            }
+         }
+      }
+
+
+      private void _load_packets_from_current_file() {
+         int curr_idx = _current_fs.Item1;
+         FileStream curr_fs = _current_fs.Item2;
+         _packets = new List<PlaybackPacket>();
+         _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+      }
+
+      private bool current_file_all_replayed {
+         get {
+            bool rv = true;
+            _packets.ForEach(p => rv = rv && p.replayed);
+            return rv;
+         }
+      }
+
+      protected override void myOnLastSubscriber() {
+         var fs = _current_fs.Item2;
+         if (fs != null) {
+            fs.Close();
+         }
+         ViewStreamDatabase.instance.removeWorker(this);
+      }
+
+      // public event PlaybackStreamProgress onProgress;
+      // public event Action onFileExhausted;
+      private PlaybackPacket _locate_packet_to_replay() {
+         foreach (var p in _packets) {
+            // returns the first non replayed packet
+            if (!p.replayed)
+               return p;
+         }
+         else
+         {
+            double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
+            myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
+            if (onProgress != null)
+               onProgress(myFileReader.Position, myFileReader.Length);
+         }
+      }
+
+      #region IWorker Members
+
+      public void doWork() {
+         if (_all_files_read)
+            return;
+         if (current_file_all_replayed) {
+            _load_next_file();
+         }
+         PlaybackPacket curr_packet = _locate_packet_to_replay();
+         if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+            byte[] ops;
+            Array row_array;
+            // what a stupid interface!
+            _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+            _fire_packet(ops, row_array);
+            curr_packet.replay_timestamp = PersistorStream.epoch_now();
+         } else {
+            PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+            // this is the time we need to fire (replay) this packet
+            long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+            long now_epoch = PersistorStream.epoch_now();
+            // fire when we are past the 
+            if (now_epoch >= time_to_fire) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+               _fire_packet(ops, row_array);
+               curr_packet.replay_timestamp = PersistorStream.epoch_now();
+            }
+         }
+      }
+
+      private void _fire_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+               //_current_table.addRows(full_table);
+               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+                                          (myLastValue as DataPacket).newRows));
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+               myFireEvent(new DataPacket(_current_table, adds, dels));
+            } else {
+               bts.Log.assert(false, $"Oprator: {op} is not expected");
+            }
+
+         }
+      }
+
+      private void _load_next_file() {
+         var new_idx = _current_fs.Item1 + 1;
+         if (new_idx >= _file_names.Count()) {
+            _all_files_read = true;
+            bts.Log.Gen.info("All files read");
+            return;
+         }
+         var prev_fs = _current_fs.Item2;
+         if (prev_fs != null)
+            prev_fs.Close();
+         var new_filename = _file_names.ElementAt(new_idx);
+         bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+         var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+         _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+         _load_packets_from_current_file();
+      }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+
+namespace BCG
+{
+   public class RecorderStream : ViewStream
+   {
+      private ViewStream mySource;
+      private DateTime myStartTime;
+      private FileStream myOutFile;
+      private string myFileName;
+      private IBlueDataSerializer mySerializer;
+            
+      public RecorderStream(ViewStream src, string recordFile)
+      {
+         mySource = src;
+         myFileName = recordFile;
+
+         myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         myStartTime = DateTime.Now;         
+         mySource.onChange += new ChangeDelegate(mySource_onChange);
+         myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+      }
+      
+      protected override void myOnLastSubscriber()
+      {
+         mySource.onChange -= mySource_onChange;
+         myOutFile.Close();
+      }
+
+      void mySource_onChange(object newValue)
+      {
+         DataPacket dp = (DataPacket)newValue;         
+         
+         BlueTable toWrite = new BlueTable(dp.schema);
+
+         if (mySerializer == null)
+         {
+            mySerializer = new UniversalSerializer(dp.schema);
+         }
+         foreach (IBlueRow r in dp.newRows)
+            toWrite.addRow(r);
+         
+         byte[] result;
+
+         if (myLastValue == null)
+            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+         else
+            mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+         BinaryWriter bw = new BinaryWriter(myOutFile);
+         //write down what time this happened
+         bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+         bw.Write(result.Length);
+         bw.Write(result);         
+         myOutFile.Flush();
+
+         myFireEvent(dp);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -1,37 +1,64 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
-using BCG;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
 
-namespace bts {
-   // this saves the updates as a Create on the same file, each update calls -save, which
-   // reqrites the whole data to the packet. The other side of this stream is the Loader
-   [DataStreamUserRespresentation("saver")]
-   public class SaverStream : ViewStream {
-      private ViewStream _stream;
-      private string _filename;
-      private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
+// using System.Runtime.Serialization.Formatters.Binary;
 
-      public SaverStream(ViewStream vs, string filename) {
-         _stream = vs;
-         _filename = filename;
-      }
+// namespace BCG
+// {
+//    [DataStreamUserRespresentation("saver")]
+//    public class SaverStream : ViewStream, ISideEffectStream
+//    {
+//       private ViewStream myStream;
+//       private string myFileName;
 
-      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
+//       public class Exception : System.Exception
+//       {
+//          public Exception(string msg) : base(msg) { }
+//       }
 
-      void _handle_changes(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         _save(dp);
-         myFireEvent(dp_obj);
-      }
+//       public SaverStream(ViewStream vs, string fileName)
+//       {
+//          if (fileName == null)
+//             throw new SaverStream.Exception("File name cannot be null");
+//          myStream = vs;
+//          myFileName = fileName;
 
-      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
+//       }
 
-      protected void _save(DataPacket dp) {
-         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
-         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
-         StreamSerializationUtils.write_to_file(str_bytes, _filename);
-      }
-   }
-}
+//       protected override void myOnFirstSubscriber()
+//       { myStream.onChange += new ChangeDelegate(myHandleChanges); }
+
+//       void myHandleChanges(object newValue)
+//       {
+//          DataPacket dp = newValue as DataPacket;
+//          mySave(dp);
+//          myFireEvent(newValue);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
+
+//       protected void mySave(DataPacket dp)
+//       {
+//          for (int i = 0; i < 5 ; i++)
+//             try
+//             {
+//                using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
+//                {
+//                   BinaryFormatter bf = new BinaryFormatter();
+//                   BlueTable t = new BlueTable(dp.schema);
+//                   foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
+//                      t.addRow(new BlueRow(r));
+//                   bf.Serialize(fs, t);
+//                }
+//                break;
+//             }
+//             catch (IOException)
+//             {
+//                System.Threading.Thread.Sleep(5);
+//             }
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -1,129 +1,115 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Runtime.Serialization.Formatters.Binary;
+using BCG.Utilities.General;
+using bts.collections;
+using System.Reflection.Metadata;
 
 
 namespace BCG {
    public class SnapshotStream : ViewStream {
-      public enum SpecificSnapshot {
-         FirstOfDay,
-         LastOfDay
-      };
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       public DateTime _snapshot_time;
-//       public string _snapshot_file;
-//       public IEnumerable<PlaybackPacket> _packets;
-//       public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-//          DateTime dt;
-//          var parsed = DateTime.TryParse(snapshot_date, out dt);
-//          dt = dt.Date;
-//          bts.Log.assert(parsed);
-//          SpecificSnapshot specific_snapshot_enum;
-//          parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-//          bts.Log.assert(parsed);
-//          string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-//          ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-//          mySetRealSource(rs);
-//       }
 
-//       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-//          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-//       }
+      private BlueTable _current_table;
+      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+      public DateTime _snapshot_time;
+      public string _snapshot_file;
+      public IEnumerable<PlaybackPacket> _packets;
 
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-//          _packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
 
-//       private void _absorb_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//             } else {
-//                bts.Log.assert(false, $"Operator: {op} is not expected");
-//             }
-//          }
-//       }
+      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+         string token_directory_name = $"{directory_name}\\{token}";
+         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+      }
 
-//       protected override void myOnLastSubscriber() {
-//       }
+      protected override void myOnFirstSubscriber() {
+         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+         _packets = StreamPersistorUtils.load_packets_from_file(fs);
+         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+         if (snapshot_packet != null) {
+            foreach (PlaybackPacket p in _packets) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+               for (int i = 0; i < ops.Length; i++) {
+                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+                  byte op = ops[i];
+                  if (op == StreamOpCode.create) {
+                     IBlueRowCollection full_table = rows_args[0];
+                     _current_table = new BlueTable(full_table);
+                  } else if (op == StreamOpCode.multipleRemoveAdd) {
+                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
+                  } else {
+                     bts.Log.assert(false, $"Oprator: {op} is not expected");
+                  }
+                  if (p.idx == snapshot_packet.idx) {
+                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+                     break;
+                  }
+               }
+            }
+         }
+      }
 
-//       // public event SnapshotStream.Progress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          return null;
-//       }
+      private void _absorb_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+            } else {
+               bts.Log.assert(false, $"Operator: {op} is not expected");
+            }
+         }
+      }
 
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
+      protected override void myOnLastSubscriber() {
+      }
 
-//          }
-//       }
+      // public event SnapshotStream.Progress onProgress;
+      // public event Action onFileExhausted;
+      private PlaybackPacket _locate_packet_to_replay() {
+         foreach (var p in _packets) {
+            // returns the first non replayed packet
+            if (!p.replayed)
+               return p;
+         }
+         return null;
+      }
+
+      private void _fire_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+               //_current_table.addRows(full_table);
+               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+                                          (myLastValue as DataPacket).newRows));
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+               myFireEvent(new DataPacket(_current_table, adds, dels));
+            } else {
+               bts.Log.assert(false, $"Oprator: {op} is not expected");
+            }
+
+         }
+      }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
-using bts;
 
 
 namespace BCG {
@@ -38,20 +37,6 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
-      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
-         DateTime max_dt = DateTime.MinValue;
-         string rv = String.Empty;
-         string[] file_names = Directory.GetFiles(directory_name);
-         for (int i = 0; i < file_names.Count() - 1; i++) {
-            string fn = file_names[i];
-            var dt = parse_datetime_from_filename(fn);
-            if (dt > max_dt) {
-               max_dt = dt;
-               rv = fn;
-            }
-         }
-         return rv;
-      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,10 +1,16 @@
+using BCG;
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using System.Text.RegularExpressions;
+
 using Bts.DataStream;
 
+using mbts;
+using mbts.spark;
 using mbts.config;
 
-using System;
-using System.Text.RegularExpressions;
-
 namespace BCG.StreamWorld.Addition
 {
    [DataStreamUserRespresentation (false)]
@@ -145,6 +151,7 @@ namespace BCG
       private string myFilterString;
       private BCG.StreamWorld.Addition.FilterStream myFilterStream;
       private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
+      private HighPerformanceTimer _timer = new HighPerformanceTimer();
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -219,6 +226,7 @@ namespace BCG
 
       private void myHandleCallbacks(object newValue)
       {
+         _timer.start();
          DataPacket retVal;
          DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
@@ -247,6 +255,7 @@ namespace BCG
                if (myFilter.filter(r))
                   myLastFiredRows.addRow(r);
          }
+         _timer.stop();
          myFireEvent(retVal);
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -1,66 +1,37 @@
 using System;
-using System.Text.RegularExpressions;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
 using BCG.Utilities.General;
 
-namespace BCG {
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
-
-      public static TimeSpan parse_duration(string input) {
-         int days = 0, hours = 0, minutes = 0, seconds = 0;
-
-         // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
-         Regex regex = new Regex(@"(\d+)([dhms])");
-         MatchCollection matches = regex.Matches(input);
-
-         foreach (Match match in matches) {
-            int value = int.Parse(match.Groups[1].Value);
-            string unit = match.Groups[2].Value;
-
-            switch (unit) {
-               case "d":
-                  days = value;
-                  break;
-               case "h":
-                  hours = value;
-                  break;
-               case "m":
-                  minutes = value;
-                  break;
-               case "s":
-                  seconds = value;
-                  break;
-            }
-         }
-
-         return new TimeSpan(days, hours, minutes, seconds);
-      }
-
+namespace BCG
+{
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
+   {
       #region constructors
-
-      public TimeBasedFilter(string col_name, string row_time_to_live)
-      : this(col_name, parse_duration(row_time_to_live)) {
-
-      }
-
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
+      {
       }
 
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
+         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
+      {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+         : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
+      {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
-         myVsd = vsd;
-         _time_col = rowTimeColumnName;
-         _ttl = rowTimeToLive;
-         myFilterFrequency = filterFrequency;
-         myNextFilter = DateTime.MinValue;
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
+      {
+         myVsd               = vsd;
+         myRowTimeColumnName = rowTimeColumnName;
+         myRowTimeToLive     = rowTimeToLive;
+         myFilterFrequency   = filterFrequency;
+         myNextFilter        = DateTime.MinValue;
 
          // add worker
          myVsd.addWorker(this);
@@ -70,25 +41,32 @@ namespace BCG {
       public event FilterChangeDelegate onFilterChange;
 
       #region IStreamFilter
-      public bool filter(IBlueRow row) {
-         DateTime now = DateTime.Now;
+      public bool filter(IBlueRow row)
+      {
+         DateTime now       = DateTime.Now;
+         DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
 
-         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
-         if (now > row_time) {
-            TimeSpan span = now - row_time;
-            if (span >= _ttl) {
+         if (now > timeStamp)
+         {
+            TimeSpan span = now - timeStamp;
+
+            if (span >= myRowTimeToLive)
+            {
                return (false);
             }
          }
+
          return (true);
       }
       #endregion
 
       #region IWorker
-      public void doWork() {
+      public void doWork()
+      {
          DateTime now = DateTime.Now;
 
-         if (now > myNextFilter) {
+         if (now > myNextFilter)
+         {
             // fire event
             onFilterChange();
 
@@ -98,7 +76,8 @@ namespace BCG {
       #endregion
 
       #region IDisposable
-      public void Dispose() {
+      public void Dispose()
+      {
          // remove worker
          myVsd.removeWorker(this);
       }
@@ -106,20 +85,10 @@ namespace BCG {
 
       #region state
       private ViewStreamDatabase myVsd;
-      private string _time_col;
-      private TimeSpan _ttl;
-      private TimeSpan myFilterFrequency;
-      private DateTime myNextFilter;
+      private string             myRowTimeColumnName;
+      private TimeSpan           myRowTimeToLive;
+      private TimeSpan           myFilterFrequency;
+      private DateTime           myNextFilter;
       #endregion
    }
-   [DataStreamUserRespresentation("timefilter")]
-   public class TimeBasedFilterStream : ViewStream {
-      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
-      :
-      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
-      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
-         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
-         mySetRealSource(new FilterStream(base_stream, filter));
-      }
-   }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -4,14 +4,18 @@ using BCG.Utilities;
 using System.Collections.Generic;
 using System.Linq;
 
-namespace BCG {
-   public class GroupByStream : MethodDrivenStream {
-      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
+namespace BCG
+{
+   public class GroupByStream : MethodDrivenStream
+   {
+      private class MyFuncGroupByEvaluator : IGroupByEvaluator
+      {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
+         {
             _groupby_cols = groupby_cols.ToArray();
-            _groupby_func = groupby_func;
+            _groupby_func = groupby_func;            
          }
          public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
 
@@ -25,7 +29,7 @@ namespace BCG {
       /// Serves as  a quick way to access all rows belonging to the same group
       /// </summary>
       private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable;
+      private RowDictionaryTable myBookKeepingTable ;
       private IGroupByEvaluator myGroupByEvaluator;
       private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
       //can map a schema to another
@@ -33,98 +37,112 @@ namespace BCG {
       private int myIdGenerator = 0;
 
       public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
+         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
+      { }
 
-      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
-         : this(stream, groupByEvaluator, true) { }
+      public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
+         : this (stream, groupByEvaluator, true)
+      {}
 
-      public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
-         : base(stream) {
+      public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
+         : base (stream)
+      {
          myGroupByEvaluator = groupbyEvaluator;
          myIdGuard = idGuard;
       }
 
-      protected override void myHandleFirstCall() {
+      protected override void myHandleFirstCall ()
+      {
          mySchema = myLastArrivedPacket.schema;
-         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
-         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
+         myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
+         myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, 
             myGroupByEvaluator.groupByColumns);
-         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
-         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
-            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
-               rowGroup, new BlueTable(mySchema)));
+         GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
+         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
+         {
+            IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
+               rowGroup, new BlueTable (mySchema)));
             if (newRow != null)
-               temp.addRow(newRow, myGetGroupString(rowGroup));
+               temp.addRow (newRow, myGetGroupString (rowGroup));
          }
-         myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
+         myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
       }
 
       /// <summary>
       /// Adding need not to anything as deletion already takes care
       /// of addition
       /// </summary>
-      protected override void myHandleIncrementalAdd() { }
-
+      protected override void myHandleIncrementalAdd()   {}
+      
       /// <summary>
       /// Both deletes and adds are handled here to make the intermediate results not
       /// appear in the spitted fired DataPacket (note this is done only in GroupByStream 
       /// because of the nature of it)
       /// </summary>
-      protected override void myHandleIncrementalDelete() {
+      protected override void myHandleIncrementalDelete()   
+      {
          myFireSwitch = false;
          RowDictionaryTable delGroups, addGroups;
          //form groups for deleted and added rows
-         myInitIncrementalChangeData(out delGroups, out addGroups);
-         ArrayList handledAddGroups = new ArrayList();
+         myInitIncrementalChangeData (out delGroups, out addGroups);
+         ArrayList handledAddGroups = new ArrayList ();
          string groupRepString;
 
-         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
+         foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
+         {
             // BlueEventLog.logAssert (delGroup.Count > 0);
-            groupRepString = myGetGroupString(delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
-               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
-            handledAddGroups.Add(addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
-               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
-            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
-            myFireSwitch |= myMergeRow(newRow, groupRepString);
+            groupRepString = myGetGroupString (delGroup);
+            IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
+               new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
+            handledAddGroups.Add (addGroup);
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
+               new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
+            IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
+            myFireSwitch |= myMergeRow (newRow, groupRepString);           
          }
 
-         foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
+         foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
+         {
             // BlueEventLog.logAssert (addGroup.Count > 0);
-            if (!handledAddGroups.Contains(addGroup)) {
+            if (!handledAddGroups.Contains (addGroup))
+            {
                //this is the case when there are rows added to a group and none
                //have been removed, hence it 'escapes' from being dealt with in 
                //the loop above
 
-               groupRepString = myGetGroupString(addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
+               groupRepString = myGetGroupString (addGroup);
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
                // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
-                  new BlueTable(aggGroup.schema)));
-               myFireSwitch |= myMergeRow(newRow, groupRepString);
+               IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, 
+                  new BlueTable (aggGroup.schema)));
+               myFireSwitch |= myMergeRow (newRow, groupRepString);
             }
          }
       }
 
-      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
-         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
+      private bool myMergeRow (IBlueRow newRow, string groupRepString)
+      {
+         IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
          //BlueEventLog.logAssert (newRow != null || existingRow != null);
          if (existingRow == null && newRow == null)
             return false;
-         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
+         else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
             return false;
-         else if (existingRow == null) {
+         else if (existingRow == null)
+         {
             //row added for the first time
-            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
-            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
+            ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
+            ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
             return true;
-         } else {
-            ((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
-            ((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
-            if (newRow != null) {
-               ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
-               ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
+         }
+         else
+         {
+            ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
+            ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
+            if (newRow != null)
+            {
+               ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
+               ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
             }
             return true;
          }
@@ -136,68 +154,77 @@ namespace BCG {
       /// </summary>
       /// <param name="delGroups"></param>
       /// <param name="addGroups"></param>
-      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
+      private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
+      {
          delGroups =
-            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
          addGroups =
-            new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
          foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
-            myBookKeepingTable.removeRow(deletedRow);
+            myBookKeepingTable.removeRow (deletedRow);
          foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
-            myBookKeepingTable.addRow(addedRow);
-         myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
+            myBookKeepingTable.addRow (addedRow);
+         myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
       }
 
-      protected override void myOnTableClear() {
+      protected override void myOnTableClear ()
+      {
          if (myBookKeepingTable.Count == 0)
             return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
-         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
-         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
-         myBookKeepingTable.clear();
-         myGroupByEvaluator.clear();
+         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
+         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
+         myBookKeepingTable.clear ();
+         myGroupByEvaluator.clear ();
       }
 
-      private IBlueRow myEvaluate(DataPacket dp) {
-         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
-         return evalRow == null ? null :
+      private IBlueRow myEvaluate (DataPacket dp)
+      {
+         IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
+         return evalRow == null ? null : 
             myIdGuard ?
-            BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
+            BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
       }
 
-      private string myGetGroupString(IBlueRowCollection group) {
-         return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
+      private string myGetGroupString (IBlueRowCollection group)
+      {
+         return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override  void myOnLastSubscriber () 
+      {
          //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream 
          //gets resubscribed, we start the process of building data again, and treat the callback as the first 
          //callback.
-         myInit();
-         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+         myInit ();
+         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
       }
 
-      protected override void myInit() {
-         base.myInit();
-         if (myBookKeepingTable != null) {
+      protected override void myInit ()
+      {
+         base.myInit ();
+         if (myBookKeepingTable != null)
+         {
             //to avoid calling init before the first callback
-            myBookKeepingTable.clear();
-            myGroupByEvaluator.clear();
+            myBookKeepingTable.clear ();
+            myGroupByEvaluator.clear ();
          }
       }
 
-      public static string getRepString(DataPacket dp, string[] colNames) {
-         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
+      public static string getRepString (DataPacket dp, string[] colNames)
+      {
+         return BlueDataUtility.rowToString (getRepRow (dp), colNames);
       }
 
-      public static IBlueRow getRepRow(DataPacket dp) {
+      public static IBlueRow getRepRow (DataPacket dp)
+      {
          if (dp.newRows.Count > 0)
-            return BlueDataUtility.extractRow(dp.newRows, 0);
+            return BlueDataUtility.extractRow (dp.newRows, 0);
          else if (dp.addedRows.Count > 0)
-            return BlueDataUtility.extractRow(dp.addedRows, 0);
+            return BlueDataUtility.extractRow (dp.addedRows, 0);
          else if (dp.deletedRows.Count > 0)
-            return BlueDataUtility.extractRow(dp.deletedRows, 0);
+            return BlueDataUtility.extractRow (dp.deletedRows, 0);
          // BlueEventLog.logAssert (false, "Illegal Data Packet");
          return null;
       }
@@ -210,21 +237,22 @@ namespace BCG {
       /// <param name="newValue"></param>
       /// <param name="isAdd"></param>
       /// <returns></returns>
-      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
-         if (lastValue == null || lastValue.Equals(DBNull.Value))
+      public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
+      {
+         if (lastValue == null || lastValue.Equals (DBNull.Value))
             return newValue;
          // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
-         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
-         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
+            // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
+            // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
          Object retVal = null;
-         if (newValue.GetType().Equals(typeof(int)))
+         if (newValue.GetType().Equals (typeof(int)))
             retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
-         else if (newValue.GetType().Equals(typeof(Double)))
+         else if (newValue.GetType().Equals (typeof(Double)))
             retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
-         else if (lastValue.GetType().Equals(typeof(Decimal)))
+         else if (lastValue.GetType().Equals (typeof(Decimal)))
             retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
-         else { }
-         // BlueEventLog.logAssert (false, "Illegal type");
+         else{}
+            // BlueEventLog.logAssert (false, "Illegal type");
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,42 +1,47 @@
-using System;
-using System.Collections;
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
 
-using BCG.Utilities;
+// namespace BCG
+// {
+// 	/// <summary>
+// 	/// Computes the max for a given column
+// 	/// </summary>
+// 	public class MaxGroupByEvaluator : GroupByEvaluator
+// 	{
 
-namespace BCG {
-    /// <summary>
-    /// Computes the max for a given column
-    /// </summary>
-    public class MaxGroupByEvaluator : GroupByEvaluator {
+//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
+//       {}
 
-        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
+//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
+//          : base (colMapEpression, outputCol, groupByCols)   {}
 
-        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
-
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
-            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
-            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
-            if (maxRow != null) {
-                bts.Log.assert(currMaxRow != null);
-                bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
-                if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
-                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
-                    if (maxRow == null)
-                        myOutputTable.removeRow(hashString);
-                    else
-                        myOutputTable.addRow(maxRow, hashString);
-                }
-            }
-            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
-            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-            // otherwise if maxRow > currMaxRow then its replaced
-            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
-                myOutputTable.addRow(maxRow, hashString);
-            return myProject(myOutputTable.getRow(hashString));
-        }
-    }
-}
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
+//             myOutputTable;
+//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
+//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
+//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
+//          if (maxRow != null)
+//          {
+//             BlueEventLog.logAssert (currMaxRow != null);
+//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
+//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
+//             {
+//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
+//                if (maxRow == null)
+//                   myOutputTable.removeRow (hashString);
+//                else
+//                   myOutputTable.addRow (maxRow, hashString);
+//             }
+//          }
+//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
+//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+//          // otherwise if maxRow > currMaxRow then its replaced
+//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
+//             myOutputTable.addRow (maxRow, hashString);
+//          return myProject (myOutputTable.getRow (hashString));
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,15 +1,20 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class MaxStream : ViewStream {
-        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
-        }
-        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
-            mySetRealSource(new GroupByStream(stream,
-                                                new MaxGroupByEvaluator(outputCol, inputCols)));
-        }
+// namespace BCG
+// {
+// 	public class MaxStream : ViewStream
+// 	{
+//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
+//       {
+//       }
+// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
+// 		{
+//          mySetRealSource (new GroupByStream (stream, 
+//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
+// 		}
 
-        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
-    }
-}
+//       public MaxStream (ViewStream stream, string outputCol)
+//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
+//       {}
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,36 +1,40 @@
-using System;
+// using System;
+// using BCG.Utilities;
 
-using BCG.Utilities;
+// namespace BCG
+// {
+//    public class MinGroupByEvaluator : MaxGroupByEvaluator
+//    {
+//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
+//          : base (outputCol, groupByCols)  {}
 
-namespace BCG {
-    public class MinGroupByEvaluator : MaxGroupByEvaluator {
-        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
-           : base(outputCol, groupByCols) { }
+//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
+//          base (colMapEpression, outputCol, groupByCols)  {}
 
-        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
-           base(colMapEpression, outputCol, groupByCols) { }
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
+//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
+//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
+//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
+//          if (minRow != null)
+//          {
+//             // BlueEventLog.logAssert (currMinRow != null);
+//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
+//             {
+//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
+//                if (minRow == null)
+//                   myOutputTable.removeRow (stringRep);
+//                else
+//                   myOutputTable.addRow (minRow, stringRep);
+//             }
+//          }
+//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
+//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
+//             myOutputTable.addRow (minRow, stringRep);
+//          return myOutputTable.getRow (stringRep);
+//       }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
-            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
-            if (minRow != null) {
-                // BlueEventLog.logAssert (currMinRow != null);
-                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-                if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
-                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
-                    if (minRow == null)
-                        myOutputTable.removeRow(stringRep);
-                    else
-                        myOutputTable.addRow(minRow, stringRep);
-                }
-            }
-            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
-            if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
-                myOutputTable.addRow(minRow, stringRep);
-            return myOutputTable.getRow(stringRep);
-        }
-
-    }
-}
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,13 +1,17 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class MinStream : ViewStream {
-        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
-            mySetRealSource(new GroupByStream(stream,
-                              new MinGroupByEvaluator(outputCol, inputCol)));
-        }
+// namespace BCG
+// {
+//    public class MinStream : ViewStream
+//    {
+//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
+//       {
+//          mySetRealSource (new GroupByStream (stream, 
+//                            new MinGroupByEvaluator (outputCol, inputCol)));
+//       }
 
-        public MinStream(ViewStream stream, string outputCol)
-                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
-    }
-}
+//       public MinStream (ViewStream stream, string outputCol)
+//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
+//       {}
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
-
-using bts;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
+using mbts.spark;
 
 namespace BCG
 {
@@ -12,7 +14,7 @@ namespace BCG
       private INonIncGroupByEvaluator _groupby_func;
       private BlueRowHasher _row_hasher;
       private string[] _groupby_cols;
-      private Stopwatch _timer = new();
+      private HighPerformanceTimer _timer = new HighPerformanceTimer();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
       public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
@@ -31,27 +33,21 @@ namespace BCG
       {
          DataPacket dp = nv as DataPacket;
          ///grouping together
-         _timer.Restart();
+         _timer.start();
          IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         _timer.stop();
+         // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
-         _timer.Restart();
+         _timer.start();
          IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
+         _timer.stop();
          _n_total_fires++;         
-         if (applyElapsedMs >= 50)
+         if (_timer.lastIntervalTime > 50)
          {
             _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
+            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
          }         
          myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
-      }
+      }      
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,82 +1,94 @@
-using System;
+// using System;
 
-namespace BCG {
-    /// <summary>
-    /// Computes the sum for a given column
-    /// </summary>
-    public class SumGroupByEvaluator : GroupByEvaluator {
-        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Computes the sum for a given column
+//    /// </summary>
+//    public class SumGroupByEvaluator : GroupByEvaluator
+//    {
+//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
+//       {}
 
-        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
+//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
+//          : base (colMapEpression, outputCol, groupByCols)   {}
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-
-            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-            return sumRow;
-        }
-    }
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
+         
+//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+//          return sumRow;
+//       }
+//    }
 
 
-    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
-        private string[] myOutputCols, myGroupByCols;
-        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
-            myOutputCols = spaceSeperatedOutputCols.Split(' ');
-            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-        }
+//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
+//    {
+//       private string[] myOutputCols, myGroupByCols;
+//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
+//       {
+//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
+//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+//       }
 
-        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
-            myOutputCols = output_columns;
-            myGroupByCols = groupby_columns;
-        }
+//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
+//       {
+//          myOutputCols = output_columns;
+//          myGroupByCols = groupby_columns;
+//       }
 
-        #region IGroupByEvaluator Members
+//       #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
+//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
 
-        public string[] groupByColumns { get { return myGroupByCols; } }
+//       public string[] groupByColumns { get { return myGroupByCols; } }
 
-        public void clear() {/*stateless*/}
+//       public void clear()  {/*stateless*/}
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
 
-        #endregion
-    }
+//       #endregion
+//    }
 
-    public class SumGroupByEvaluator<T> : GroupByEvaluator {
-        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-           : base(outputCol, groupByCols) { }
+//    public class SumGroupByEvaluator<T> : GroupByEvaluator
+//    {
+//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+//          : base(outputCol, groupByCols)
+//       { }
 
-        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
+//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+//          : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+//       public override IBlueRow evaluate(DataPacket dp)
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-            return sumRow;
-        }
-    }
+//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+//          return sumRow;
+//       }
+//    }
 
-    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
-        private string[] myOutputCols, myGroupByCols;
-        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
-            myOutputCols = spaceSeperatedOutputCols.Split(' ');
-            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-        }
+//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
+//    {
+//       private string[] myOutputCols, myGroupByCols;
+//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
+//       {
+//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
+//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+//       }
 
-        #region IGroupByEvaluator Members
+//       #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
+//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
 
-        public string[] groupByColumns { get { return myGroupByCols; } }
+//       public string[] groupByColumns { get { return myGroupByCols; } }
 
-        public void clear() {/*stateless*/}
+//       public void clear()  {/*stateless*/}
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
 
-        #endregion
-    }
+//       #endregion
+//    }
 
-}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,89 +1,107 @@
-using System;
-using System.Collections.Generic;
+// using System;
+// using System.Collections.Generic;
 
-namespace BCG {
-    public class NullMapper : IStreamMapper {
-        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+// namespace BCG
+// {
+//    public class NullMapper : IStreamMapper
+//    {
+//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-        public NullMapper(string[] valid_cols) {
-            foreach (string c in valid_cols)
-                _valid_cols[c] = true;
-            _valid_cols["id"] = true;
-        }
+//       public NullMapper(string[] valid_cols)
+//       {
+//          foreach (string c in valid_cols)
+//             _valid_cols[c] = true;
+//          _valid_cols["id"] = true;
+//       }
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-        public IBlueRow map(IBlueRow row) {
-            MutableBlueRow out_row = new MutableBlueRow(row);
-            foreach (string c in row.schema.columnNames)
-                if (!_valid_cols.ContainsKey(c))
-                    out_row.setValue(c, DBNull.Value);
-            return out_row;
-        }
-    }
+//       public IBlueRow map(IBlueRow row)
+//       {
+//          MutableBlueRow out_row = new MutableBlueRow(row);
+//          foreach (string c in row.schema.columnNames)
+//             if (!_valid_cols.ContainsKey(c))
+//                out_row.setValue(c, DBNull.Value);
+//          return out_row;
+//       }
+//    }
 
-    public class SumStream : ViewStream {
-        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
-            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-        }
+//    public class SumStream : ViewStream
+//    {
+//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
+//       {
+//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+//          mySetRealSource(new GroupByStream(stream,
+//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+//       }
 
-        public SumStream(ViewStream stream, string[] output_columns) {
-            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-            rs = new ProjectStream(rs, "!fake_col");
-            mySetRealSource(rs);
-        }
+//       public SumStream(ViewStream stream, string[] output_columns)
+//       {
+//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+//          rs = new ProjectStream(rs, "!fake_col");
+//          mySetRealSource(rs);
+//       }
 
-        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
-            string[] output_columns = space_seperated_output_columns.Split(' ');
-            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
-            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
-        }
+//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
+//       {
+//          string[] output_columns = space_seperated_output_columns.Split(' ');
+//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
+//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
+//       }
 
-        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
-            ViewStream rs = null;
-            groupby_columns = groupby_columns ?? new string[] { };
-            if (groupby_columns.Length == 0) {
-                // rs = new AddColumnsStream(stream, "fake_col=0");
-                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
-                // rs = new ProjectStream(rs, "!fake_col");
-            } else {
-                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-            }
-            if (force_null_into_other_columns) {
-                List<string> valid_cols = new List<string>(output_columns);
-                valid_cols.AddRange(groupby_columns);
-                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-            }
-            mySetRealSource(rs);
-        }
+//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
+//       {
+//          ViewStream rs = null;
+//          groupby_columns = groupby_columns ?? new string[] {};
+//          if (groupby_columns.Length == 0)
+//          {
+//             rs = new AddColumnsStream(stream, "fake_col=0");
+//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
+//             rs = new ProjectStream(rs, "!fake_col");
+//          }
+//          else
+//          {
+//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+//          }
+//          if (force_null_into_other_columns)
+//          {
+//             List<string> valid_cols = new List<string>(output_columns);
+//             valid_cols.AddRange(groupby_columns);
+//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+//          }
+//          mySetRealSource(rs);
+//       }
 
-        public SumStream(ViewStream stream, string outputCol)
-        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
-        }
-    }
+//    public SumStream (ViewStream stream, string outputCol)
+//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
+//    {
+//    }
+// }
 
-    public class IntSumStream : ViewStream {
-        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
-            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-        }
+// public class IntSumStream : ViewStream
+// {
+//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
+//    {
+//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+//       mySetRealSource(new GroupByStream(stream,
+//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
+//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+//    }
 
-    }
+// }
 
 
-    public class SumStream<T> : ViewStream {
-        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
-            string[] inputCols = spaceSeparatedInputCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
-        }
-    }
-}
+//    public class SumStream<T> : ViewStream
+//    {
+//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
+//       {
+//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
+//          mySetRealSource(new GroupByStream(stream,
+//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -1,79 +0,0 @@
-using BCG;
-
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-using ZstdSharp.Unsafe;
-
-namespace bts {
-    public class RowDiffStream : ViewStream {
-        private ViewStream _vs;
-        private readonly string _order_col, _col_1, _col_2;
-        private readonly int _diff;
-        private IBlueTableSchema _out_schema;
-        private readonly string _diff_col_name;
-        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
-            _vs = vs;
-            _order_col = order_col;
-            _col_1 = col_1;
-            _col_2 = col_2;
-            _diff = diff;
-            _diff_col_name = diff_col_name;
-        }
-
-        private string _diff_col_name_perc {
-            get {
-                return $"{_diff_col_name}_percentage";
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _vs.onChange += _handle_vs;
-        }
-
-        protected override void myOnLastSubscriber() {
-            _vs.onChange -= _handle_vs;
-        }
-
-        private void _handle_vs(object dp_obj) {
-            var dp = dp_obj as DataPacket;
-            IBlueRowCollection nrows = dp.newRows;
-            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
-            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
-            List<IBlueRow> row_list = new List<IBlueRow>();
-            foreach (IBlueRow r in nrows)
-                row_list.Add(r);
-            //OrderBy sorts in ascedning order
-            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
-            var sorted_row_array = sorted_rows.ToArray();
-            // reverse it to make it descending    
-            sorted_rows.Reverse();
-            //sort rows so we can do the diff
-            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
-            int i = 0;
-            foreach (IBlueRow r in sorted_rows) {
-                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
-                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
-                // we assume x - y, and name variables like that
-                double x = r.get<double>(_col_1);
-                int y_row_idx = i + _diff;
-                double diff = System.Double.NaN;
-                double perc = System.Double.NaN;
-                if (y_row_idx < sorted_rows.Count()) {
-                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
-                    diff = x - y;
-                    perc = 100.0 * diff / y;
-                }
-                nr.setValue(_diff_col_name, diff);
-                nr.setValue(_diff_col_name_perc, perc);
-                rv.addRow(nr);
-                i++;
-            }
-
-            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs

@@ -27,8 +27,7 @@ namespace BCG
             string prependString = isLeft ? "left" : "right";
             return String.Concat (prependString, colName).GetHashCode ();
          }
-         public override bool Equals (object? obj)  =>
-            GetHashCode() == obj?.GetHashCode();
+         public override bool Equals (object obj)  {return GetHashCode () == obj.GetHashCode ();}
       }
 
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs

@@ -10,9 +10,9 @@ namespace BCG
 {
    public class JoinStream : ViewStream
    {
-      protected InternalJoinStream? myIjs;
-      protected ProjectStream? myPs;
-      protected ColumnMap? myLeftCm, myRightCm;
+      protected InternalJoinStream myIjs;
+      protected ProjectStream myPs;
+      protected ColumnMap myLeftCm, myRightCm;
       protected bool myIsProject = true;
       private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
       

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,10 +1,10 @@
-using bts;
-
-using mbts;
-
-using System;
+using System;
 using System.Collections.Generic;
-using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
+using mbts.spark;
 
 namespace BCG
 {
@@ -40,7 +40,8 @@ namespace BCG
 
       public void _on_stream_cb(object dp_obj)
       {
-         var timer = Stopwatch.StartNew();
+         HighPerformanceTimer timer = new HighPerformanceTimer();
+         timer.start();
          DataPacket dp = (DataPacket)dp_obj;
          if (_schema == null)
          {
@@ -66,12 +67,13 @@ namespace BCG
                   _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
             }
          }
-         timer.Stop();
+         timer.stop();
          _n_total_fires++;
-         if (timer.ElapsedMilliseconds >= 50) {
+         if (timer.lastIntervalTime > 50)
+         {
             _n_expensive_fires++;
-            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
-               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
+            string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
          }
          if (on_new_data != null)
             on_new_data();

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -1,10 +1,10 @@
-using bts;
-
-using mbts;
-
-using System;
+using System;
 using System.Collections.Generic;
-using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
+using mbts.spark;
 
 namespace BCG
 {
@@ -114,7 +114,8 @@ namespace BCG
          if (_is_ready())
          {
             _construct_schema();
-            var timer = Stopwatch.StartNew();
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
             IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
             List<object[]> output_rows = new List<object[]>();
 
@@ -137,12 +138,13 @@ namespace BCG
                }
             }
             IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.Stop();
+            timer.stop();
             _n_total_fires++;
-            if (timer.ElapsedMilliseconds >= 250) {
+            if (timer.lastIntervalTime >= 250)
+            {
                _n_expensive_fires++;
-               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
+               string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
             }
             myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
@@ -152,12 +154,15 @@ namespace BCG
       {
          if (_output_mapper != null)
          {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
             IBlueRowCollection output_rows;
-            var timer = Stopwatch.StartNew();
+            timer.start();
             output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.Stop();
-            if (timer.ElapsedMilliseconds >= 50) {
-               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
+            timer.stop();
+            if (timer.lastIntervalTime >= 50)
+            {
+               // string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
+               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
             }
             return output_rows;
          }

src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs

@@ -1,7 +1,11 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Text;
+using BCG.Utilities.General;
+using BCG.Utilities;
 using mbts;
+using mbts.spark;
 using Spark.Core.Collections;
 
 namespace BCG
@@ -222,8 +226,7 @@ namespace BCG
 
       #region IDictionary Members
 
-      public void Add(object key, object? value) =>
-         throw new NotSupportedException("IBlueRow is readonly");
+      public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
 
       public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
 
@@ -253,9 +256,10 @@ namespace BCG
          }
       }
 
-      public object? this[object key] {
-         get => key is string str ? this[str] : this[(int)key];
-         set => throw new NotSupportedException("IBlueRow is readonly");
+      public object this[object key]
+      {
+         get { return key is string ? this[key.ToString()] : this[(int)key]; }
+         set { throw new NotSupportedException("IBlueRow is readonly"); }
       }
 
       #endregion
@@ -290,6 +294,7 @@ namespace BCG
       protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
                                           myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
       private bool myManageUnmappedLeftRows = false;
+      protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
       protected string myLeftProj = String.Empty, myRightProj = String.Empty;
       protected IBlueTableSchema myLeftSchema, myRightSchema;
       protected JoinProjectHelper myJoinProjectHelper;
@@ -356,6 +361,7 @@ namespace BCG
 
       private void myHandleStreamChanges(DataPacket dp, bool isLeft)
       {
+         myTimer.start();
          myInitSchema(dp.schema, isLeft);
          BlueTable dels = null, adds = null;
          if (myState != null)
@@ -365,6 +371,7 @@ namespace BCG
          }
          myDeleteRows(dp.deletedRows, isLeft, myState, dels);
          myAddRows(dp.addedRows, isLeft, myState, adds);
+         myTimer.stop();
          if (myState != null)
          {
             myFireEvent(new DataPacket(myState, adds, dels));

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -1,31 +0,0 @@
-namespace BCG {
-    using System.Collections.Generic;
-
-    public class KeyValueToRelationalStream : ViewStream {
-        private readonly ViewStream _kv_str;
-        private readonly string _doc_id_col, _key_col, _val_col;
-        public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
-            _kv_str = kv_str;
-            _doc_id_col = doc_id_col;
-            _key_col = key_col;
-            _val_col = val_col;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _kv_str.onChange += _on_str_update;
-        }
-
-        protected override void myOnLastSubscriber() {
-            _kv_str.onChange -= _on_str_update;
-        }
-
-        private void _on_str_update(object dp_obj) {
-            DataPacket dp = dp_obj as DataPacket;
-            var rows = dp.newRows;
-            var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
-            var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-        }
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs

@@ -91,8 +91,7 @@ namespace BCG
          }
       }
 
-      public void Add (object key, object? value) =>
-         throw new NotSupportedException ("The IBlueRow is readonly");
+      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
 
       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
 

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -1,64 +1,51 @@
 using System;
 using System.Collections;
-using System.Collections.Generic;
-
 using BCG;
 using BCG.Utilities;
 
-namespace BCG {
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
+namespace BCG
+{
+   public class LiteralStream : ViewStream
+   {
+      public LiteralStream(string tableName, string schema, string value)
+      {
+         string [] columns = schema.Split(' ');
 
 
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
+         ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id",0,typeof(int));
          int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
+         foreach (string c in columns)
+         {
+            string [] columnAndType = c.Split('/');
             if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
+               ci[n] = new ColumnInfo(c,n,typeof(double));
             else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
+               ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
             n++;
          }
 
 
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
+         string [] rows = value.Split(';');
          n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
+         foreach (string r in rows) 
+            try 
+            {
+               if(r == string.Empty)
                   continue;
                t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
+                  BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));   
+            }
+            catch (Exception e)
+            {
                throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
             }
 
          mySetRealSource(new BlueRowCollectionStaticStream(t));
       }
 
-      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
-
-      // public LiteralStream(string value) {
-
-      // }
+      public LiteralStream(string schema, string value) : this ("literal",schema, value)
+      {}
    }
 }

src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs

@@ -1,24 +1,24 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
-//    {
-//       IBlueTableSchema mySchema;
+namespace BCG
+{
+   public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
+   {
+      IBlueTableSchema mySchema;
 
-//       public IBlueRow map (IBlueRow row)
-//       {
-//          if (mySchema == null)
-//          {
-//             ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
-//             int i = 0;
-//             foreach (ColumnInfo ci in row.schema.columnInfo)
-//                cis[i++] = ci.colType == null ?
-//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
-//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
-//             mySchema = new BlueTableSchema (row.schema.tableName, cis);
-//          }
-//          return new BlueRow (mySchema, row);
-//       }
-//    }
-// }
+      public IBlueRow map (IBlueRow row)
+      {
+         if (mySchema == null)
+         {
+            ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
+            int i = 0;
+            foreach (ColumnInfo ci in row.schema.columnInfo)
+               cis[i++] = ci.colType == null ?
+                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
+                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
+            mySchema = new BlueTableSchema (row.schema.tableName, cis);
+         }
+         return new BlueRow (mySchema, row);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -1,10 +1,15 @@
+using System;
+using System.Data;
+using System.Collections;
+
 using Bts.DataStream;
 
+using mbts;
+using mbts.spark;
 using mbts.config;
 
-using System;
-
-namespace BCG {
+namespace BCG
+{
 
 
    /// <summary>
@@ -16,45 +21,24 @@ namespace BCG {
    [DataStreamUserRespresentation (false)]
    public class MapStream : MethodDrivenStream
    {
+      private HighPerformanceTimer t = new HighPerformanceTimer ();
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
 
-      public class FuncWrapper : IStreamMapper {
-         private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func;
-         Func<IBlueRow, IBlueRow> _row_map_func;
-
-         public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
-         Func<IBlueRow, IBlueRow> row_map_func) {
-            _schema_map_func = schema_map_func;
-            _row_map_func = row_map_func;
-         }
-         public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-            return _schema_map_func(input_schema);
-         }
-         public IBlueRow map(IBlueRow row) {
-            return _row_map_func(row);
-         }
-
-      }
-
-      public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
+      public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
+      {
          if (myUseNewMap)
-            mySetRealSource(new SuperMapStream(vs, mapper));
+            mySetRealSource (new SuperMapStream (vs, mapper));
          else
             myMapper = mapper;
       }
-
-      public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
-                                       Func<IBlueRow, IBlueRow> row_map_func)
-                                       :
-                                       this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
-      }
-
-      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
+      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
+      {
          if (myUseNewMap)
             mySetRealSource(new SuperMapStream(vs, mapper));
-         else {
+         else
+         {
             mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
             myChangeableMapper = mapper;
          }
@@ -63,68 +47,77 @@ namespace BCG {
       /// <summary>
       /// Gets or sets the value of the mapper. Setting the value will result in re
       /// </summary>
-      private void myMapperChanged() {
+      private void myMapperChanged () 
+      {
          if (myLastArrivedPacket != null)
-            myChangeHandler(new DataPacket(myLastArrivedPacket.newRows,
+            myChangeHandler (new DataPacket (myLastArrivedPacket.newRows, 
                                              myLastArrivedPacket.newRows,
                                              myLastArrivedPacket.newRows));
       }
 
 
-      protected override void myHandleFirstCall() {
-
-         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
-         myLastFiredPacket = new DataPacket();
-         myLastFiredPacket.newRows = new BlueTable(outputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable(outputSchema);
+      protected override void myHandleFirstCall ()
+      {
+         
+         IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
+         myLastFiredPacket = new DataPacket ();
+         myLastFiredPacket.newRows = new BlueTable (outputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
          foreach (IBlueRow row in myLastArrivedPacket.newRows)
-            ((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row));
+            ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
          myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
       }
 
-      protected override void myHandleIncrementalAdd() {
+      protected override void myHandleIncrementalAdd()
+      {
          //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
-            IBlueRow newRow = myMapper.map(row);
-            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
+         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows)
+         {
+            IBlueRow newRow = myMapper.map (row);
+            (myLastFiredPacket.newRows as BlueTable).addRow (newRow);
+            (myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
          }
          //t.stop ();
          //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
          //System.Diagnostics.Trace.WriteLine (" ");
       }
 
-      protected override void myHandleIncrementalDelete() {
+      protected override void myHandleIncrementalDelete()
+      {  
          //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
+         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
+         {
             IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
+            (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
+            (myLastFiredPacket.newRows as BlueTable).deleteRow (row);
          }
          //t.stop ();
          //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
       }
+     
 
-
-      protected override void myOnFirstSubscriber() {
-         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
+      protected override void myOnFirstSubscriber() 
+      {
+         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
       }
 
-      protected override void myOnLastSubscriber() {
-         myInit();
+      protected override  void myOnLastSubscriber () 
+      {
+         myInit ();
          if (myStream != null)
-            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
       }
 
-      protected override void myOnTableClear() {
+      protected override void myOnTableClear ()
+      {
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-         myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
-         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
       }
 
-
+      
    }
 
    /*

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -1,13 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
-
 using BCG.Utilities;
 using BCG.Utilities.General;
-
 using mbts;
 
-namespace BCG {
+namespace BCG
+{
    /// <summary>
    /// SuperMap gives the user the ability
    /// to map one row to one or more rows
@@ -18,11 +17,13 @@ namespace BCG {
    /// can chage itself and raise an event causing
    /// a remapping to happen
    /// </summary>
-   public class SuperMapStream : ViewStream {
+   public class SuperMapStream : ViewStream
+   {
       #region super mapper that wraps around stream mapper
-      private class MyMappingWrapper : ISuperStreamMapper {
+      private class MyMappingWrapper : ISuperStreamMapper
+      {
          private IStreamMapper myMapper;
-
+         
          public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
 
          #region ISuperStreamMapper Members
@@ -31,7 +32,8 @@ namespace BCG {
 
          public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
 
-         public event MapperChangeDelegate onMapperChange {
+         public event MapperChangeDelegate onMapperChange
+         {
             add { }
             remove { }
          }
@@ -40,18 +42,21 @@ namespace BCG {
       }
       #endregion
 
-      private class MyFuncWrapper : ISuperStreamMapper {
+      private class MyFuncWrapper : ISuperStreamMapper
+      {
          private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
          private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
          public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
-                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) {
+                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
+         {
             _get_schema = get_schema;
             _map_rows = map_rows;
          }
 
          public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
 
-         public IBlueRow[] map(IBlueRow r) {
+         public IBlueRow[] map(IBlueRow r)
+         {
             IEnumerable<IBlueRow> rows = _map_rows(r);
             return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
          }
@@ -71,15 +76,19 @@ namespace BCG {
 
       public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
                                             Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, new MyFuncWrapper(get_schema, map_rows)) { }
+         : this(vs, new MyFuncWrapper(get_schema, map_rows))
+      { }
 
       public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, s => out_schema, map_rows) { }
+         : this(vs, s => out_schema, map_rows)
+      { }
 
       public SuperMapStream(ViewStream vs, IStreamMapper mapper)
-         : this(vs, new MyMappingWrapper(mapper)) { }
+         : this(vs, new MyMappingWrapper(mapper))
+      { }
 
-      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) {
+      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
+      {
          myStream = vs;
          myMapper = mapper;
          myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
@@ -87,7 +96,8 @@ namespace BCG {
 
       protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          myStream.onChange -= new ChangeDelegate(myHandleData);
          myRows = null;
          myBaseRows = null;
@@ -95,7 +105,8 @@ namespace BCG {
          //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
       }
 
-      private void myHandleData(object newValue) {
+      private void myHandleData(object newValue)
+      {
          DataPacket idp = newValue as DataPacket;
          //HighPerformanceTimer timer = new HighPerformanceTimer();
          //timer.start();
@@ -106,19 +117,24 @@ namespace BCG {
          BlueTable dels = new BlueTable(myRows.schema),
                      adds = new BlueTable(myRows.schema);
 
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
             List<IBlueRow> mappedRows = myMap[(int)r["id"]];
-            foreach (IBlueRow mr in mappedRows) {
+            foreach (IBlueRow mr in mappedRows)
+            {
                dels.addRow(mr);
                myRows.deleteRow(mr);
             }
             myMap.Remove((int)r["id"]);
          }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
             IBlueRow[] mappedRows = myMapper.map(r);
-            foreach (IBlueRow mr in mappedRows) {
+            foreach (IBlueRow mr in mappedRows)
+            {
                IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"])) {
+               if (!myMap.ContainsKey((int)r["id"]))
+               {
                   myMap.Add((int)r["id"], new List<IBlueRow>());
                }
                myMap[(int)r["id"]].Add(idChangedMapRow);
@@ -129,12 +145,13 @@ namespace BCG {
          //timer.stop();
          // if (timer.lastIntervalTime > 20)
          //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
-         myFireEvent(new DataPacket(myRows, adds, dels));
+         // myFireEvent(new DataPacket(myRows, adds, dels));
       }
 
-      private void myHandleMapChange() {
+      private void myHandleMapChange()
+      {
          if (myBaseRows != null)
-            myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
+            myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs

@@ -1,142 +0,0 @@
-using BCG;
-using System;
-using System.IO;
-using MongoDB.Bson;
-using BCG.Utilities.General;
-using System.Linq;
-using System.Collections.Generic;
-using System.Runtime.Intrinsics.X86;
-
-
-namespace bts {
-   // listens to the underlying stream, and writes each seriialized packet in a 
-   // seperate file
-   [DataStreamUserRespresentation("dp_recorder")]
-   public class DataPacketRecorderStream : ViewStream {
-      private ViewStream _base_stream;
-      private readonly string _folder;
-      StreamIOSerializationController _serializer = new StreamIOSerializationController();
-      private UInt32 _n_updates = 0;
-      public DataPacketRecorderStream(ViewStream base_stream, string folder) {
-         _base_stream = base_stream;
-         _folder = folder;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_update;
-      }
-
-      private void _handle_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         byte[] ser_data = null;
-         if (_n_updates == 0) {
-            StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
-            ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
-         } else {
-            StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
-            ser_data = _serializer.serialize(new StreamIOMessage(mra));
-         }
-         Log.assert(ser_data != null);
-         _n_updates++;
-         string file_name = $"{_folder}\\{_n_updates}.bin";
-         _write_to_file(ser_data, file_name);
-         myFireEvent(dp);
-      }
-
-      private static void _write_to_file(byte[] data, string file_name) {
-         using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
-            // Write the byte array to the file
-            fs.Write(data, 0, data.Length);
-         }
-      }
-   }
-   [DataStreamUserRespresentation("dp_replay")]
-   public class DataPacketReplayStream : ViewStream, IWorker {
-      private ViewStreamDatabase _vsd;
-      private readonly string _folder_path;
-      private readonly TimeSpan _fire_period;
-      IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
-      private int _file_ptr = 0;
-      private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
-      private BlueTable _state = null;
-
-      public DataPacketReplayStream(string folder_path, string fire_period)
-      :
-      this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
-      }
-
-      public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
-         _vsd = vsd;
-         _folder_path = folder_path;
-         _fire_period = fire_period;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         MasterWorker.getInstance(false).addWorker(this, _fire_period);
-         _load_all_files();
-      }
-
-      protected override void myOnLastSubscriber() {
-         MasterWorker.getInstance(false).removeWorker(this);
-      }
-
-      public void doWork() {
-         if (_file_ptr >= _file_data.Count())
-            // we can set _file_ptr to 0 and keep going as well
-            return;
-         var curr_d = _file_data[_file_ptr++];
-         var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
-         StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
-         Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
-         switch (msg.op) {
-            case StreamIOMessage.StreamOp.CREATE: {
-                  _state = new BlueTable(msg.create_message.rows);
-                  break;
-               }
-            case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
-                  var mra = msg.multiple_remove_add_message;
-                  foreach (IBlueRow r in mra.removed_rows)
-                     _state.deleteRow(r);
-                  foreach (IBlueRow r in mra.added_rows)
-                     _state.addRow(r);
-                  break;
-               }
-         }
-         var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
-         myFireEvent(dp);
-      }
-
-      private void _load_all_files() {
-         Func<string, string> extract_file_name = (string full_path) => {
-            string fn = full_path.Split("\\").Last();
-            fn = fn.Split(".").First();
-            return fn;
-
-         };
-         string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
-
-         // Iterate over the .bin files and perform an action
-         foreach (string file in bin_files) {
-            byte[] file_data = File.ReadAllBytes(file);
-            _file_data.Add(Tuple.Create(file, file_data));
-         }
-         _file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
-            string fn1 = extract_file_name(t1.Item1);
-            string fn2 = extract_file_name(t2.Item1);
-            int fn_i1 = Int32.Parse(fn1);
-            int fn_i2 = Int32.Parse(fn2);
-            if (fn_i1 < fn_i2)
-               return -1;
-            else if (fn_i1 == fn_i2)
-               return 0;
-            else if (fn_i1 > fn_i2)
-               return 1;
-            return 0;
-         });
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -1,227 +0,0 @@
-using BCG;
-using BCG.Utilities.General;
-using System.Collections.Generic;
-
-//using MongoDB.Driver;
-
-using System;
-using System.IO;
-using System.Linq;
-
-namespace bts
-{
-    public class PubSubTestingStream : ViewStream, IWorker
-    {
-        public class PayloadMessage
-        {
-            public int msg_id = 0;
-            public string guid;
-            public Int64 travel_epoch_ms = 0;
-            public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
-            {
-                msg_id = msg_id_args;
-                travel_epoch_ms = send_epoch_args;
-                payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++)
-                {
-                    payload[i] = payload_byte;
-                }
-            }
-            private PayloadMessage() { }
-            public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
-            { }
-
-            public byte[] serialize()
-            {
-                var str = new MemoryStream();
-                BinaryWriter bw = new BinaryWriter(str);
-                bw.Write(msg_id);
-                bw.Write(guid);
-                bw.Write(travel_epoch_ms);
-                bw.Write(payload.Length);
-                bw.Write(payload);
-                str.Position = 0;
-                var bytes = str.ToArray();
-                return bytes;
-            }
-
-            public static PayloadMessage deserialize(byte[] ser_bytes)
-            {
-                var str = new MemoryStream(ser_bytes);
-                BinaryReader br = new BinaryReader(str);
-                int msg_id = br.ReadInt32();
-                string guid = br.ReadString();
-                Int64 epoch = br.ReadInt64();
-                int len = br.ReadInt32();
-                var payload = br.ReadBytes(len);
-                return new PayloadMessage()
-                {
-                    msg_id = msg_id,
-                    guid = guid,
-                    travel_epoch_ms = epoch,
-                    payload = payload
-                };
-            }
-
-        }
-        //public void
-        private readonly int _msg_size;
-        private readonly TimeSpan _publish_period;
-        private DateTime _last_publish_time = DateTime.MinValue;
-        private PublisherCore _pub_core = new PublisherCore();
-        private BlueTable _table;
-        private int _n_pubs = 0;
-        private readonly int _max_pubs;
-        private int _n_updates = 0;
-        private bool _continue_sub_thread = true;
-        private ViewStreamDatabase _vsd;
-        private bool _fire_dirty = false;
-        private PayloadMessage _last_sent_msg, _last_recvd_msg;
-        private readonly int _payload_sz;
-        private const byte _payload_byte = (byte)'A';
-        private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
-        {
-            _vsd = vsd;
-            _payload_sz = payload_size;
-            var schema = BlueDataUtility.form_schema("PubSubTest",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("n_pubs", TypeCode.Int32),
-            Tuple.Create("n_recvs", TypeCode.Int32),
-            Tuple.Create("last_sent_id", TypeCode.Int32),
-            Tuple.Create("last_recv_id", TypeCode.Int32),
-            Tuple.Create("last_sent_guid", TypeCode.String),
-            Tuple.Create("last_recv_guid", TypeCode.String),
-            Tuple.Create("match", TypeCode.Boolean),
-            Tuple.Create("echo_time", TypeCode.Int32));
-            _table = new BlueTable(schema);
-            _publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
-            _msg_size = payload_size;
-            _max_pubs = max_pubs;
-            _tag_name = tag_name;
-        }
-
-        public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
-        : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
-
-        protected override void myOnFirstSubscriber()
-        {
-            _vsd.addWorker(this);
-            System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
-            th.IsBackground = true;
-            th.Start();
-        }
-
-        protected override void myOnLastSubscriber()
-        {
-            _vsd.removeWorker(this);
-            _continue_sub_thread = false;
-        }
-
-        void _log_updates(IEnumerable<PubSubServerMessage> updates)
-        {
-            var idx = 0;
-            foreach (var msg in updates)
-            {
-                bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
-                idx++;
-            }
-        }
-
-        private void _subscribe_thread_func()
-        {
-            // subscription is done in another thread
-            SubscriberCore sub_core = new SubscriberCore();
-            sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread)
-            {
-                // TODO recv_id
-                var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
-                if (updates == null || updates.Count() == 0)
-                    continue;
-                _log_updates(updates);
-                _n_updates += updates.Count();
-                if (updates.Count() > 0)
-                {
-                    var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0)
-                    {
-                        _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
-                        _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    }
-                    else
-                    {
-                        // why are we getting empty or null data ? (server is sending data regardless?)
-                    }
-                    _fire_dirty = true;
-                }
-            }
-        }
-
-        private void _publish_next_packet()
-        {
-            if (_last_sent_msg == null)
-            {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
-                {
-                    guid = Guid.NewGuid().ToString()
-                };
-            }
-            else
-            {
-                _last_sent_msg.msg_id++;
-                _last_sent_msg.guid = Guid.NewGuid().ToString();
-                _last_sent_msg.travel_epoch_ms = epoch_now();
-            }
-            var ser_bytes = _last_sent_msg.serialize();
-            _n_pubs++;
-            _pub_core.publish("Blob", _tag_name, ser_bytes);
-            _fire_dirty = true;
-        }
-
-        private void _fire_dp()
-        {
-            object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
-            object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
-            object echo_time = DBNull.Value;
-            object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
-            object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null)
-            {
-                echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
-            }
-            var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
-            IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
-                                        last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
-            BlueTable table = new BlueTable(_table.schema);
-            table.addRow(nr);
-            var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        public void doWork()
-        {
-            if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs)
-                {
-                    _publish_next_packet();
-                    _last_publish_time = DateTime.Now;
-                }
-            // publish 
-            // TODO: send id
-            if (_fire_dirty)
-            {
-                _fire_dirty = false;
-                _fire_dp();
-            }
-        }
-
-        public static long epoch_now()
-        {
-            return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-        }
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -1,223 +0,0 @@
-using BCG;
-using BCG.Utilities.General;
-using System.Collections.Generic;
-
-
-using System;
-using System.IO;
-using System.Linq;
-using System.Runtime.CompilerServices;
-using bts.api;
-
-namespace bts
-{
-   [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker
-   {
-      public class PayloadMessage
-      {
-         public readonly string msg_id;
-         public readonly Int64 timestamp = 0;
-         public readonly byte[] payload = null;
-
-         public static byte[] create_data(int payload_size, byte payload_byte)
-         {
-            var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++)
-            {
-               rv[i] = payload_byte;
-            }
-            return rv;
-         }
-
-         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
-         // {
-         //    guid = guid_;
-         //    timestamp = timestamp_;
-         //    payload = new byte[payload_size];
-         //    for (int i = 0; i < payload_size; i++)
-         //    {
-         //       payload[i] = payload_byte;
-         //    }
-         // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
-         {
-            msg_id = guid_;
-            timestamp = timestamp_;
-            payload = payload_;
-         }
-
-         private PayloadMessage() { }
-         public byte[] serialize()
-         {
-            var str = new MemoryStream();
-            BinaryWriter bw = new BinaryWriter(str);
-            bw.Write(msg_id);
-            bw.Write(timestamp);
-            bw.Write(payload.Length);
-            bw.Write(payload);
-            str.Position = 0;
-            var bytes = str.ToArray();
-            return bytes;
-         }
-
-         public static PayloadMessage deserialize(byte[] ser_bytes)
-         {
-            var str = new MemoryStream(ser_bytes);
-            BinaryReader br = new BinaryReader(str);
-            string guid = br.ReadString();
-            Int64 timestamp_ = br.ReadInt64();
-            int len = br.ReadInt32();
-            var payload = br.ReadBytes(len);
-            return new PayloadMessage(guid, timestamp_, payload);
-         }
-      }
-      //public void
-      private readonly int _msg_size;
-      private readonly TimeSpan _publish_period;
-      private DateTime _last_publish_time = DateTime.MinValue;
-      private PublisherCore _pub_core;
-      private ISubscriberCore _sub_core;
-      private BlueTable _table;
-      private int _n_pubs = 0;
-      private readonly int _max_pubs;
-      private int _n_updates = 0;
-      private ViewStreamDatabase _vsd;
-      private PayloadMessage _last_sent_msg, _last_recvd_msg;
-      private const byte _payload_byte = (byte)'A';
-      private readonly string _tag_name;
-      private readonly int _payload_sz;
-      private readonly IPubSubRouter _router;
-      private enum IOState
-      {
-         SEND = 0,
-         RECV = 1
-      };
-      IOState _state = IOState.SEND;
-
-      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
-      {
-         var route = new PubSubRoute(server_url, ".*");
-         _router = new PubSubRouter(route);
-         _vsd = vsd;
-         _payload_sz = payload_size;
-         var schema = BlueDataUtility.form_schema("PubSubTest",
-         Tuple.Create("id", TypeCode.Int32),
-         Tuple.Create("sent_id", TypeCode.String),
-         Tuple.Create("sent_time", TypeCode.Int64),
-         Tuple.Create("recv_id", TypeCode.String),
-         Tuple.Create("recv_time", TypeCode.Int64),
-         Tuple.Create("match", TypeCode.Boolean),
-         Tuple.Create("echo_time", TypeCode.Int32));
-         _table = new BlueTable(schema);
-         _publish_period = publish_period;
-         _tag_name = tag_name;
-      }
-
-      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
-      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
-
-      protected override void myOnFirstSubscriber()
-      {
-         _pub_core = new PublisherCore(_router);
-         _sub_core = new SubscriberCore(_router);
-         _sub_core.subscribe(_tag_name);
-         _vsd.addWorker(this);
-
-      }
-
-      protected override void myOnLastSubscriber()
-      {
-         _vsd.removeWorker(this);
-         _sub_core.unsubscribe(_tag_name);
-      }
-
-      private void _fire_dp_after_send()
-      {
-         // as soon as the data is published, its reflected in the stream state
-         Log.assert(_last_sent_msg != null);
-         BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
-            DBNull.Value, DBNull.Value);
-         _table.addRow(r);
-         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-
-      private void _fire_dp_after_recv()
-      {
-         Log.assert(_last_recvd_msg != null);
-         // update the last publish row
-         int id = _table.Count - 1;
-         IBlueRow last_row = _table[id];
-         _table.deleteRow(last_row);
-         var recv_ts = epoch_now();
-         BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
-                                 String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
-         _table.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-
-      private string _get_next_id()
-      {
-         return _table.Count.ToString();
-      }
-
-      private void _publish_next_packet()
-      {
-         var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
-         _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
-         var ser_bytes = _last_sent_msg.serialize();
-         _n_pubs++;
-         _pub_core.publish("Blob", _tag_name, ser_bytes);
-         _last_publish_time = DateTime.Now;
-      }
-
-      public void doWork()
-      {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
-         {
-            foreach (var u in updates)
-            {
-               var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id)
-               {
-                  return recv_msg;
-               }
-            }
-            return null;
-         };
-         switch (_state)
-         {
-            case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period)
-               {
-                  _publish_next_packet();
-                  _last_publish_time = DateTime.Now;
-                  _fire_dp_after_send();
-                  _state = IOState.RECV;
-                  return;
-               }
-               break;
-            case IOState.RECV:
-               var updates = _sub_core.get_updates(TimeSpan.Zero);
-               updates = updates.Where(u => u.tag == _tag_name);
-               var recv_msg = find_packet(updates);
-               if (recv_msg != null)
-               {
-                  _last_recvd_msg = recv_msg;
-                  _fire_dp_after_recv();
-                  _state = IOState.SEND;
-                  break;
-               }
-               break;
-         }
-      }
-
-      public static long epoch_now()
-      {
-         return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs

@@ -1,58 +0,0 @@
-namespace bts {
-   using BCG;
-   using BCG.Utilities.General;
-
-   using System;
-
-   public class SimpleTCPEchoStream : ViewStream, IWorker {
-      private string _ip;
-      private int _port;
-      private SimpleTCPClient _tcp_client;
-      private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
-      Tuple.Create("id", TypeCode.Int32),
-      Tuple.Create("event", TypeCode.String));
-      private BlueTable _table;
-      private SyncQueue<string> _action_queue = new SyncQueue<string>();
-
-      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
-      : this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
-
-      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
-         _ip = ip;
-         _port = port;
-         _table = new BlueTable(_schema);
-         _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         ViewStreamDatabase.instance.addWorker(this);
-         _tcp_client.register_listener((msg) => { _on_msg(msg); });
-         _tcp_client.start();
-      }
-
-      private void _on_msg(string msg) {
-         _action_queue.enqueue(msg);
-      }
-
-      public void doWork() {
-         if (_action_queue.getCount() == 0)
-            return;
-         BlueTable added_rows = new BlueTable(_schema);
-         var sz = _action_queue.getCount();
-         for (int i = 0; i < sz; i++) {
-            string msg = _action_queue.dequeue();
-            if (msg == null)
-               break;
-            IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
-            _table.addRow(nr);
-            added_rows.addRow(nr);
-         }
-         if (added_rows.Count == 0)
-            return;
-         DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
-         myFireEvent(dp);
-      }
-
-
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs

@@ -1,77 +0,0 @@
-using System.Net.Sockets;
-using System.Net;
-
-using BCG;
-using System;
-using System.Collections.Generic;
-using MongoDB.Bson;
-using BCG.Utilities.General;
-using System.Linq;
-
-
-namespace bts {
-   // this class, a little specialized, takes the first snapshot thats fired 
-   // from the base stream sorts them by a given column, and fires one row at a time 
-   public class SnapshotReplayStream : ViewStream, IWorker {
-
-      private ViewStreamDatabase _vsd;
-      private ViewStream _snapshot_stream;
-      private readonly string _sort_col;
-      private readonly int _buffer_size;
-      private TimeSpan _fire_period;
-      private int _in_row_ptr = 0;
-      private List<IBlueRow> _in_rows = new List<IBlueRow>();
-      private CircularBufferTable _out_table;
-
-
-      public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period)
-      : this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { }
-
-      public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) {
-         _vsd = vsd;
-         _snapshot_stream = base_str;
-         _sort_col = sort_col;
-         _buffer_size = buffer_size;
-         _fire_period = fire_period;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _snapshot_stream.onChange += _handle_stream_update;
-         MasterWorker.getInstance(false).addWorker(this, _fire_period);
-      }
-
-      protected override void myOnLastSubscriber() {
-         base.myOnLastSubscriber();
-      }
-
-      private void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         if (_in_rows.Count > 0)
-            return;
-         _out_table = new CircularBufferTable(dp.schema, _buffer_size);
-         // _in_rows is filled up only once
-         foreach (IBlueRow r in dp.newRows) {
-            _in_rows.Add(r);
-         }
-         _in_rows.Sort((IBlueRow r1, IBlueRow r2) => {
-            return BlueDataUtility.compare_rows(r1, r2, _sort_col);
-         });
-      }
-
-      public void doWork() {
-         if (_in_row_ptr >= _in_rows.Count) {
-            // we can loop around or not here
-            return;
-         }
-         if (_in_rows.Count > 0) {
-            // fire row
-            IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
-            _out_table.add_row(curr_row);
-            DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
-               myLastValue == null ? null : myLastValue as DataPacket, false);
-            myFireEvent(dp);
-         }
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -1,809 +0,0 @@
-using bts;
-
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Reflection;
-
-namespace BCG {
-
-    /// <summary>
-    /// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
-    /// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
-    /// </summary>
-    [DataStreamUserRespresentation("tests")]
-    public class TesStream : ViewStream {
-        static byte[] create_byte_array() {
-            Random random = new Random();
-            var sz = random.Next(1, 1000);// % 1000;
-
-            // Generate a random byte
-            byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
-            byte[] rv = new byte[sz];
-            for (int i = 0; i < sz; i++) {
-                rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
-            }
-            return rv;
-        }
-
-        private IBlueTableSchema _schema;
-        class TestResult {
-            public string test_name = MethodBase.GetCurrentMethod()?.Name
-               ?? throw new InvalidOperationException("Failed to get test name from method");
-            public bool test_result;
-        }
-        private List<Func<TestResult>> _tests = new
-         List<Func<TestResult>>();
-
-        public TesStream() {
-            _schema = BlueDataUtility.form_schema("Tests",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("test", TypeCode.String),
-            Tuple.Create("result", TypeCode.Boolean),
-            Tuple.Create("exception", TypeCode.String)
-            );
-            // register tests here
-            _tests.Add(new Func<TestResult>(_run_sub_msg_test));
-            _tests.Add(new Func<TestResult>(_run_pub_msg_test));
-            _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
-            _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
-            _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
-            // _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
-            // _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
-            //  _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
-            //_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
-            // _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
-            // _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
-            // _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
-            // _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
-            //_tests.Add(new Func<TestResult>(_cpp_data_test));
-            //_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
-            _tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
-
-        }
-
-        protected override void myOnFirstSubscriber() {
-            BlueTable rows = new BlueTable(_schema);
-            // u the tests here            
-            foreach (var test in _tests) {
-                string ex_msg = String.Empty;
-                TestResult tr = new TestResult() {
-                    test_name = test.Method.Name,
-                    test_result = false
-                };
-                try {
-                    tr = test();
-                } catch (Exception ex) {
-                    ex_msg = ex.Message + "\n" + ex.StackTrace;
-                }
-                IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
-                rows.addRow(r);
-            }
-            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-        protected override void myOnLastSubscriber() {
-        }
-
-        private TestResult _run_sub_msg_test() {
-            bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
-                connection = Guid.NewGuid().ToString(),
-                subscribing_app = Guid.NewGuid().ToString(),
-                tags = new string[] { "one", "two", "three" }
-            };
-            MemoryStream str = new MemoryStream();
-            sub_msg.serialize_into_str(str);
-            str.Position = 0;
-            bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
-            sub_msg_.consruct_deserialize(str);
-            bool res = sub_msg.Equals(sub_msg_);
-            return new TestResult {
-                test_name = "SubscribeMsg",
-                test_result = res
-            };
-        }
-
-        private TestResult _run_pub_msg_test() {
-            var pub_msg = new bts.PublishMsg() {
-                connection = Guid.NewGuid().ToString(),
-                publishing_app = Guid.NewGuid().ToString(),
-                tag = Guid.NewGuid().ToString(),
-                tag_type = "DS_",
-                is_compressed = false,
-                publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-                data_to_publish = create_byte_array(),
-            };
-            MemoryStream str = new MemoryStream();
-            pub_msg.serialize_into_str(str);
-            str.Position = 0;
-            var pub_msg_ = new bts.PublishMsg();
-            pub_msg_.consruct_deserialize(str);
-            bool res = pub_msg.Equals(pub_msg_);
-            return new TestResult {
-                test_name = "PublishMsg",
-                test_result = res
-            };
-        }
-
-        private TestResult _run_pubsub_server_msg_test() {
-            var msg = new bts.PubSubServerMessage() {
-                tag = Guid.NewGuid().ToString(),
-                tag_type = Guid.NewGuid().ToString(),
-                data = create_byte_array()
-            };
-            MemoryStream str = new MemoryStream();
-            msg.serialize_into_str(str);
-            str.Position = 0;
-            var msg_ = new bts.PubSubServerMessage();
-            msg_.consruct_deserialize(str);
-            bool res = msg.Equals(msg_);
-            return new TestResult {
-                test_name = "PubSubServerMessage",
-                test_result = res
-            };
-        }
-
-        private TestResult _run_pubsub_client_msg_test() {
-            PubSubClientMessage msg = new PubSubClientMessage() {
-                msg_type = PubSubMessageType.SUBSCRIBE,
-                sub_msg = new bts.SubscribeMsg() {
-                    connection = Guid.NewGuid().ToString(),
-                    subscribing_app = Guid.NewGuid().ToString(),
-                    tags = new string[] { "one", "two", "three" }
-                }
-            };
-            var str = new MemoryStream();
-            msg.serialize_into_str(str);
-            str.Position = 0;
-            var msg_ = new PubSubClientMessage();
-            msg_.consruct_deserialize(str);
-            var res = msg.Equals(msg_);
-            return new TestResult {
-                test_name = "PubSubClientMessage",
-                test_result = res
-            };
-        }
-
-        private TestResult _run_columninfo_serialization_test() {
-            Func<ColumnInfo, bool> test_ci = (ci) => {
-                MemoryStream str = new MemoryStream();
-                BinaryWriter bw = new BinaryWriter(str);
-                StreamSerializationUtils.serialize_column_info(ci, ref bw);
-                str.Position = 0;
-                //var ci_txt = BlueDataUtility.to_string(ci);
-                //var ci_ser = 
-                //bts.Log.Gen.info("ci: {}")
-                BinaryReader br = new BinaryReader(str);
-                var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
-                return BlueDataUtility.compute_equality(ci, ci_prime);
-            };
-            int i = 0;
-            foreach (Type t in BlueDataUtility.valid_column_types()) {
-                ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
-                i++, t);
-                var res = test_ci(ci);
-                if (!res) {
-                    return new TestResult {
-                        test_name = "ColumnInfoSerialization",
-                        test_result = false
-                    };
-                }
-            }
-            return new TestResult {
-                test_name = "ColumnInfoSerialization",
-                test_result = true
-            };
-        }
-
-
-        private TestResult _run_schema_serialization_test() {
-            var table_name = "test_table";
-            var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
-            IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
-            MemoryStream str = new MemoryStream();
-            BinaryWriter bw = new BinaryWriter(str);
-            StreamSerializationUtils.serialize_schema(schema, ref bw);
-            _write_to_file(str, @"c:\Temp\tests\schema.bin");
-            var hex_txt = StreamSerializationUtils.to_hex(str);
-            bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
-
-            str.Position = 0;
-            BinaryReader br = new BinaryReader(str);
-            var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
-            var res = BlueDataUtility.compute_equality(schema, schema_prime);
-
-            return new TestResult {
-                test_name = "SchemaSerialization",
-                test_result = res
-            };
-        }
-
-
-
-
-        private TestResult _run_datetime_seriliaztion_test() {
-            DateTime dt = DateTime.Now;
-            DateTimeOffset dt_off = new DateTimeOffset(dt);
-            long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
-            DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
-            DateTime dt_p = dt_off_pr.DateTime;
-            var passes = dt.Equals(dt_p);
-            return new TestResult {
-                test_name = "_run_datetime_seriliaztion_test",
-                test_result = false
-            };
-        }
-
-        private TestResult _run_full_stream_serialization_test() {
-            // create message
-            var schema = create_schema_with_all_types();
-            BlueTable cr_rows = new BlueTable(schema);
-            // 100 rows
-            int n_rows = 100;
-            for (int i = 0; i < n_rows; i++) {
-                var nr = generate_random_row(schema, i);
-                cr_rows.addRow(nr);
-            }
-            MemoryStream str = new MemoryStream();
-            BinaryWriter bw = new BinaryWriter(str);
-            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
-            bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
-            UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
-            ser.serialize(stream_io_msg, ref bw);
-            str.Position = 0;
-            BinaryReader br = new System.IO.BinaryReader(str);
-            UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
-            var stream_io_msg_prime = deser.deserialize(ref br);
-            bool passes = stream_io_msg.Equals(stream_io_msg_prime);
-            var n_dels = 50;
-            BlueTable del_rows = new BlueTable(cr_rows.schema);
-            foreach (IBlueRow r in cr_rows) {
-                del_rows.addRow(r);
-                if (del_rows.Count == n_dels)
-                    break;
-            }
-            IBlueRowCollection added_rows = del_rows;
-            bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
-            str = new MemoryStream();
-            bw = new BinaryWriter(str);
-            stream_io_msg = new bts.StreamIOMessage(mra_msg);
-            ser.serialize(stream_io_msg, ref bw);
-            bw.BaseStream.Position = 0;
-            br = new BinaryReader(str);
-            //deser = new UniversalStreamIODeserializer();
-            stream_io_msg_prime = deser.deserialize(ref br);
-            passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
-            return new TestResult {
-                test_name = "UniversalSerialization",
-                test_result = passes
-            };
-        }
-
-        private TestResult _run_full_stream_serialization_with_controller_test() {
-            // create message
-            var schema = create_schema_with_all_types();
-            BlueTable cr_rows = new BlueTable(schema);
-            // 100 rows
-            int n_rows = 100;
-            for (int i = 0; i < n_rows; i++) {
-                var nr = generate_random_row(schema, i);
-                cr_rows.addRow(nr);
-            }
-            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
-            bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
-            IStreamIOMsgSerializer ser = new StreamIOSerializationController();
-            var ser_bytes = ser.serialize(stream_io_msg);
-            IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
-            var stream_io_msg_prime = deser.deserialize(ser_bytes);
-            bool passes = stream_io_msg.Equals(stream_io_msg_prime);
-            //mult remove adds
-            // remove all rows and add 50 more
-
-            BlueTable del_rows = new BlueTable(cr_rows);
-            BlueTable added_rows = new BlueTable(cr_rows.schema);
-            var n_adds = 50;
-            for (int i = 0; i < n_adds; i++) {
-                var nr = generate_random_row(schema, i);
-                added_rows.addRow(nr);
-            }
-            bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
-            stream_io_msg = new bts.StreamIOMessage(mra_msg);
-            ser_bytes = ser.serialize(stream_io_msg);
-            var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
-            Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
-            stream_io_msg_prime = deser.deserialize(ser_bytes);
-            passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
-            return new TestResult {
-                test_name = "UniversalSerializationController",
-                test_result = passes
-            };
-        }
-
-        private TestResult _row_serialization_tests_no_dbnulls() {
-            // IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
-            //                                                     new Tuple<string, TypeCode>("id", TypeCode.Int32),
-            //                                                     new Tuple<string, TypeCode>("i1", TypeCode.Int32),
-            //                                                     new Tuple<string, TypeCode>("d1", TypeCode.Double),
-            //                                                     new Tuple<string, TypeCode>("string_1", TypeCode.String));
-            // bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
-            // IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
-            var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
-            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
-            MemoryStream str = new MemoryStream();
-            IBlueRow r = generate_random_row(schema, 0, false);
-            BinaryWriter bw = new BinaryWriter(str);
-            row_ser.serialize_row(r, 0, ref bw);
-            _write_to_file(str, @"C:\Temp\tests\row.bin");
-            str.Position = 0;
-            byte[] raw_byte_array = str.ToArray();
-            var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
-            Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
-            BinaryReader br = new BinaryReader(str);
-            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
-            IBlueRow r_prime = des.deserialize_row(ref br);
-            string row_str = r_prime.to_string();
-            Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
-            var passes = BlueDataUtility.compute_equality(r, r_prime);
-            return new TestResult {
-                test_name = "UniversalRowSerialization_NonNull",
-                test_result = passes
-            };
-        }
-
-        private TestResult _deserialize_pub_msg_test() {
-            var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
-            PubSubClientMessage ps_client_msg = new PubSubClientMessage();
-            ps_client_msg.consruct_deserialize(pub_msg_ser);
-            bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
-            return new TestResult {
-                test_name = "_deserialize_pub_msg_test",
-                test_result = passes
-            };
-        }
-
-        private TestResult _cpp_data_test() {
-            //create and write schema
-            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
-                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
-                                                                new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
-                                                                new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
-                                                                new Tuple<string, TypeCode>("d_1", TypeCode.Double),
-                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String)
-                                                                );
-            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
-            UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
-            StreamIOSerializationController ser_controller = new StreamIOSerializationController();
-            StreamIODeserializationController deser_controller = new StreamIODeserializationController();
-
-            IBlueTableSchema schema_p = null;
-            var schema_file = @"c:\Temp\tests\schema.bin";
-            var row_file = @"C:\Temp\tests\row.bin";
-            var create_op_file = @"C:\Temp\tests\create_op.bin";
-            var mra_op_file = @"C:\Temp\tests\mra_op.bin";
-            IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
-
-            //serialize schema, write it to a file, read it back, deserialize it and match schemas
-            byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
-            _write_to_file(ser_schema_bytes, schema_file);
-
-            byte[] schema_file_bytes = _read_from_file(schema_file);
-            schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
-            bool passes = BlueDataUtility.compute_equality(schema, schema_p);
-
-            //serialize row, write it to a file, read it back, deserialize it and match schemas    
-            byte[] ser_row_bytes = row_ser.serialize_row(row);
-            _write_to_file(ser_row_bytes, row_file);
-
-            byte[] row_file_bytes = _read_from_file(row_file);
-            var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
-            Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
-
-            IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
-            passes = passes && BlueDataUtility.compute_equality(row, row_p);
-
-            {
-                // serialize a create operation
-                int n_rows = 100;
-                BlueTable table = new BlueTable(schema);
-                for (int i = 0; i < n_rows; i++) {
-                    IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
-                    table.addRow(nr);
-                }
-                StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
-                //StreamIOSerializationController ser_controller = new StreamIOSerializationController();
-                byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
-                _write_to_file(create_ser_bytes, create_op_file);
-
-                // read it and make sure its right
-                byte[] create_op_file_bytes = _read_from_file(create_op_file);
-                //StreamIODeserializationController deser_controller = new StreamIODeserializationController();
-                var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
-                _write_to_file(create_ser_bytes, mra_op_file);
-
-                passes = passes && cr_msg.Equals(cr_msg_p);
-
-            }
-            //multiple remove adds
-            {
-                // remove all rows and add them again
-                int n_rows = 100;
-                BlueTable table = new BlueTable(schema);
-                for (int i = 0; i < n_rows; i++) {
-                    IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
-                    table.addRow(nr);
-                }
-                StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
-
-                //StreamIOSerializationController ser_controller = new StreamIOSerializationController();
-                byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
-                _write_to_file(mra_ser_bytes, mra_op_file);
-
-                // reasd it and make sure it matches
-                byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
-                //StreamIODeserializationController deser_controller = new StreamIODeserializationController();
-                StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
-                passes = passes && mra_msg.Equals(mra_msg_p);
-            }
-            return new TestResult {
-                test_name = "_cpp_data_test",
-                test_result = passes
-            };
-        }
-
-        private TestResult _write_stream_updates_to_files() {
-            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
-                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
-                                                                new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
-                                                                new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
-                                                                new Tuple<string, TypeCode>("d_1", TypeCode.Double),
-                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String)
-                                                                );
-            var packets_path = @"C:\Temp\tests\stream_packets\";
-            Func<long> now_epochs = () => {
-                DateTimeOffset now = DateTimeOffset.UtcNow;
-                // Convert to Unix time in milliseconds
-                long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
-                return unix_time_milliseconds;
-            };
-            Func<int, bool, string> curr_file_path = (id, create) => {
-                //var epoch_ms = now_epochs();
-                var file_path = $"{packets_path}{id}.bin";
-                if (create)
-                    file_path = $"{packets_path}create_{id}.bin";
-                return file_path;
-            };
-            // write 1000 stream updates
-            {
-                int init_sz = 100;
-                BlueTable table = new BlueTable(schema);
-                StreamIOSerializationController ser_controller = new StreamIOSerializationController();
-                // create table of 100 rows
-                for (int i = 0; i < init_sz; i++) {
-                    IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
-                    table.addRow(nr);
-                }
-                // write create msg
-                StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
-                byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
-                var create_file_path = curr_file_path(table.Count, true);
-                _write_to_file(create_ser_bytes, create_file_path);
-
-                // add 1000 rows and write an io msg for each update and write it to a file
-                int n_updates = 1000;
-                int period_ms = 50;
-
-                // write a create
-                for (int i = 0; i < n_updates; i++) {
-                    // addition
-                    BlueTable adds = new BlueTable(schema);
-                    int id = table.Count;
-                    IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
-                    adds.addRow(nr);
-                    BlueTable dels = new BlueTable(schema);
-                    table.addRow(nr);
-                    StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
-                    byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
-                    var mra_file_path = curr_file_path(id, false);
-                    _write_to_file(mra_ser_bytes, mra_file_path);
-
-                    System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
-                }
-                // now start writing increments and add 1 row at a time
-            }
-            return new TestResult {
-                test_name = "_write_updates_as_files",
-                test_result = true
-            };
-        }
-
-
-        private TestResult _row_serialization_tests_with_dbnulls() {
-            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
-                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
-                                                                new Tuple<string, TypeCode>("i1", TypeCode.Int32),
-                                                                new Tuple<string, TypeCode>("d1", TypeCode.Double),
-                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String));
-            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
-            IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
-            MemoryStream str = new MemoryStream();
-            BinaryWriter bw = new BinaryWriter(str);
-            row_ser.serialize_row(r, 0, ref bw);
-            str.Position = 0;
-            BinaryReader br = new BinaryReader(str);
-            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
-            IBlueRow r_prime = des.deserialize_row(ref br);
-            var passes = BlueDataUtility.compute_equality(r, r_prime);
-            return new TestResult {
-                test_name = "UniversalRowSerialization_WithDBNulls",
-                test_result = passes
-            };
-        }
-
-        private TestResult _row_serialization_tests_all_types() {
-            var schema = create_schema_with_all_types();
-            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
-            IBlueRow r = generate_random_row(schema, 0, false);
-            MemoryStream str = new MemoryStream();
-            BinaryWriter bw = new BinaryWriter(str);
-            row_ser.serialize_row(r, 0, ref bw);
-            _write_to_file(str, @"C:\Temp\tests\row.bin");
-            str.Position = 0;
-            BinaryReader br = new BinaryReader(str);
-            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
-            IBlueRow r_prime = des.deserialize_row(ref br);
-            var passes = BlueDataUtility.compute_equality(r, r_prime);
-            return new TestResult {
-                test_name = "UniversalRowSerialization_AllTypes",
-                test_result = passes
-            };
-        }
-
-        private static TestResult _run_pubsub_server_stream_tag_handler_test() {
-            // create message
-            var schema = create_schema_with_all_types();
-            var tag = "DS_" + schema.tableName;
-            BlueTable cr_rows = new BlueTable(schema);
-            // 100 rows
-            int n_rows = 100;
-            for (int i = 0; i < n_rows; i++) {
-                var nr = generate_random_row(schema, i);
-                cr_rows.addRow(nr);
-            }
-
-            PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
-            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
-            bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
-            IStreamIOMsgSerializer ser = new StreamIOSerializationController();
-            byte[] msg_bytes = ser.serialize(str_io_msg);
-            handler.update(tag, msg_bytes);
-
-            //now get it back for the handler
-            byte[] content = handler.get_content(tag);
-            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
-            bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
-            // test here
-            var passes = str_io_msg.Equals(str_io_msg_p);
-            // delete all rows
-            BlueTable del_rows = new BlueTable(cr_rows);
-            cr_rows.clear();
-            // added 50 new rows
-            BlueTable added_rows = new BlueTable(del_rows.schema);
-            int new_state_sz = 50;
-            for (int i = 0; i < new_state_sz; i++) {
-                var nr = generate_random_row(schema, i);
-                added_rows.addRow(nr);
-            }
-            cr_rows.addRows(added_rows);
-            // the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
-            // 1. generate op
-            bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
-            str_io_msg = new bts.StreamIOMessage(mra);
-            // 2. serialize it into bytes
-            msg_bytes = ser.serialize(str_io_msg);
-            // 3. send it to handler
-            handler.update(tag, msg_bytes);
-            // 4. get the state from the handler
-            byte[] content_p = handler.get_content(tag);
-            // 5. deserialize the state into a stream op
-            str_io_msg_p = deser.deserialize(content_p);
-            // 6. check to make sure the state's rows are good
-            passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
-                    BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
-            return new TestResult {
-                test_result = passes,
-                test_name = "run_pubsub_server_stream_tag_handler_test"
-            };
-        }
-
-
-        private static TestResult _run_single_tag_data_test_clean() {
-            // create message
-            var test_name = MethodBase.GetCurrentMethod().Name;
-            var schema = create_schema_with_all_types();
-            var tag = "DS_" + schema.tableName;
-            BlueTable table = new BlueTable(schema);
-            IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
-            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
-            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
-            // create with 100 rows
-            int n_rows = 100;
-            bool is_succ = true;
-            for (int i = 0; i < n_rows; i++) {
-                var nr = generate_random_row(schema, i);
-                // perform the same action SingletagData, and verify the states match
-                PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
-                table.addRow(nr);
-                st_data.handle_publish(pub_msg);
-                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
-                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
-            }
-            // change all rows
-            for (int i = 0; i < n_rows; i++) {
-                var nr = generate_random_row(schema, i);
-                var dr = table[i];
-                table.deleteRow(dr);
-                table.addRow(nr);
-                PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
-                st_data.handle_publish(pub_msg);
-                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
-                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
-            }
-            // delete all rows
-            var aux_table = new BlueTable(table);
-            foreach (IBlueRow r in aux_table) {
-                table.deleteRow(r);
-                PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
-                st_data.handle_publish(pub_msg);
-                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
-                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
-            }
-            return new TestResult() {
-                test_result = is_succ,
-                test_name = test_name
-            };
-        }
-
-
-
-        #region Helper functions
-
-        private static void _write_to_file(byte[] ser_bytes, string file_name) {
-            _write_to_file(new MemoryStream(ser_bytes), file_name);
-        }
-
-        private static void _write_to_file(MemoryStream str, string file_name) {
-            if (String.IsNullOrEmpty(file_name))
-                return;
-            byte[] str_bytes = str.ToArray();
-            using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
-                // Write the byte array to the file
-                fs.Write(str_bytes, 0, str_bytes.Length);
-            }
-        }
-
-        private static byte[] _read_from_file(string file_path) {
-            var rv = File.ReadAllBytes(file_path);
-            return rv;
-        }
-
-        public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
-            var content = st_data.get_current_content();
-            var msg_p = deser.deserialize(content);
-            Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
-            return msg_p.create_message.rows;
-
-        }
-
-        public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
-            ref IStreamIOMsgSerializer serializer) {
-            Func<IBlueTableSchema> get_schema = () => {
-                return (added_row != null ? added_row : removed_row).schema;
-            };
-            bts.StreamIOMessage str_io_msg = null;
-            if (is_create) {
-                Log.assert(added_row != null);
-                var aux_new = new BlueTable(added_row.schema);
-                aux_new.addRow(added_row);
-                var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
-                str_io_msg = new bts.StreamIOMessage(cr_msg);
-            } else {
-                var schema = get_schema();
-                BlueTable added_table = new BlueTable(schema);
-                if (added_row != null)
-                    added_table.addRow(added_row);
-                BlueTable rem_table = new BlueTable(schema);
-                if (removed_row != null)
-                    rem_table.addRow(removed_row);
-                bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
-                    new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
-                str_io_msg = new bts.StreamIOMessage(mra_msg);
-            }
-            byte[] msg_bytes = serializer.serialize(str_io_msg);
-            PublishMsg pub_msg = new PublishMsg() {
-                connection = "test_connection",
-                tag = $"DS_{get_schema()}",
-                tag_type = "DS",
-                data_to_publish = msg_bytes
-            };
-            return pub_msg;
-        }
-
-        public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
-            excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
-            var table_name = "test_table";
-            BlueTableSchema schema = new BlueTableSchema();
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            foreach (Type t in BlueDataUtility.valid_column_types()) {
-                if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
-                    continue;
-                var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
-                ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
-                cis.Add(ci);
-            }
-            cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
-            schema = new BlueTableSchema(table_name, cis.ToArray());
-            return schema;
-        }
-
-        public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
-            Random rnd = new Random();
-            object[] row_vals = new object[schema.columnInfo.Length];
-            foreach (ColumnInfo ci in schema.columnInfo) {
-                if (ci.colName == "id") {
-                    row_vals[ci.colNo] = id;
-                    continue;
-                }
-                object v = get_random_object(ci.type_code());
-                if (include_dbnulls && rnd.NextSingle() % 4 == 0)
-                    v = DBNull.Value;
-                row_vals[ci.colNo] = v;
-            }
-            return new BlueRow(schema, row_vals);
-        }
-
-        public static object get_random_object(TypeCode type_code) {
-            Random random = new Random();
-            switch (type_code) {
-                case TypeCode.Boolean:
-                    return random.Next(2) == 0; // Random boolean value (true or false)
-                case TypeCode.Byte:
-                    return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
-                case TypeCode.Char:
-                    return (char)random.Next(65, 65 + 27); // Random char value
-                case TypeCode.Decimal:
-                    Decimal dec = 100.123M;
-                    return dec;
-                case TypeCode.DateTime:
-                    return DateTime.Now;
-                case TypeCode.Double:
-                    return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
-                case TypeCode.Int16:
-                    return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
-                case TypeCode.Int32:
-                    return random.Next(int.MinValue, int.MaxValue); // Random int value
-                case TypeCode.Int64:
-                    return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
-                case TypeCode.SByte:
-                    return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
-                case TypeCode.Single:
-                    return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
-                case TypeCode.String:
-                    return Guid.NewGuid().ToString();
-                case TypeCode.UInt16:
-                    return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
-                case TypeCode.UInt32:
-                    return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
-                case TypeCode.UInt64:
-                    byte[] ulongBuffer = new byte[8];
-                    random.NextBytes(ulongBuffer);
-                    return BitConverter.ToUInt64(ulongBuffer, 0);
-                default:
-                    throw new ArgumentException($"Unsupported TypeCode: {type_code}");
-                    return null;
-            }
-        }
-        #endregion
-
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/TickingTestStream.cs

@@ -1,31 +0,0 @@
-namespace BCG {
-   using System;
-
-   public class TickingTestStream : ViewStream {
-      public TickingTestStream(int ticking_period_msecs)
-      :
-      this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
-
-      public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
-         ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
-         vs = new MathStream(vs, "x=id*123 y=x*x");
-         mySetRealSource(vs);
-         //vs.onChange += _on_str_update;
-      }
-
-      // private void _on_str_update(object dp_obj) {
-      //    DataPacket dp = dp_obj as DataPacket;
-      //    IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
-      //    BlueTable state = new BlueTable(out_schema);
-      //    foreach (IBlueRow r in dp.newRows) {
-      //       MutableBlueRow nr = new MutableBlueRow(out_schema, r);
-      //       DateTime time = r.get_datetime("time");
-      //       var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-      //       nr.setValue("time_txt", time_txt);
-      //       state.addRow(nr);
-      //    }
-      //    var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
-      //    myFireEvent(dp_out);
-      // }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -3,37 +3,45 @@ using Google.Protobuf.WellKnownTypes;
 using System;
 using BCG.Utilities.General;
 
-namespace BCG {
+namespace BCG
+{
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 
-   public class TimeSeriesStream : ViewStream {
+   public class TimeSeriesStream : ViewStream
+   {
       private BlueTable _table;
       private ViewStream _stream;
       private int _id = 0;
 
-      public TimeSeriesStream(ViewStream stream) {
+      public TimeSeriesStream(ViewStream stream)
+      {
          _stream = stream;
       }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _stream.onChange += new ChangeDelegate(_onstream_change);
       }
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
 
-      private void _onstream_change(object o) {
+      private void _onstream_change(object o)
+      {
          DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
-         if (_table == null) {
+         if (_table == null)
+         {
             IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
             schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
             _table = new BlueTable(schema);
          }
          BlueTable added_rows = new BlueTable(_table.schema);
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
             ar.setValue("timestamp", now);
             ar.setValue("id", _id++);
@@ -45,6 +53,114 @@ namespace BCG {
       }
    }
 
+   public class TimeBucketStream : ViewStream, IStreamMapper
+   {
+      private ViewStream _src;
+      private readonly string _time_stamp_col;
+      private IBlueTableSchema _schema;
+      private int _bucket_size_seconds;
+
+      public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
+      {
+         _time_stamp_col = time_stamp_col;
+         _bucket_size_seconds = bucket_size_seconds;
+         var vs = new MapStream(input, this);
+         mySetRealSource(vs);
+      }
+
+      #region IStreamMapper Members
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
+      {
+         IBlueTableSchema rv = input_schema;
+         //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
+         rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
+         return rv;
+
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         DateTime ts = row.get<DateTime>(_time_stamp_col);
+         TimeSpan time_of_day = (ts - ts.Date);
+         int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
+         double buckets = seconds / _bucket_size_seconds;
+         int bucket_number = Convert.ToInt32(Math.Floor(buckets));
+         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
+         rv.setValue("bucket_number", bucket_number);
+         return rv;
+      }
+
+      #endregion
+   }
+
+   // public class ThrottleStream : ViewStream, IWorker
+   // {
+   //    private ViewStreamDatabase _vsd;
+   //    private ViewStream _stream;
+   //    private int _throttle_ms;
+   //    private TimeSpan _min_fire_period;
+   //    private DateTime _last_Fire_time = DateTime.MinValue;
+   //    private DataPacket _unfired_dp;
+   //    public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
+   //    {
+   //       _vsd = vsd;
+   //       _stream = base_stream;
+   //       _throttle_ms = throttle_ms;
+   //       _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
+   //    }
+
+   //    protected override void myOnFirstSubscriber()
+   //    {
+   //       _vsd.addWorker(this);
+   //       _stream.onChange += new ChangeDelegate(_onstream_change);
+   //    }
+   //    protected override void myOnLastSubscriber()
+   //    {
+   //       _vsd.removeWorker(this);
+   //       _stream.onChange -= new ChangeDelegate(_onstream_change);
+   //    }
+
+   //    private void _onstream_change(object obj)
+   //    {
+   //       DataPacket dp = obj as DataPacket;
+   //       if (DateTime.Now - _last_Fire_time >= _min_fire_period)
+   //       {
+   //          // ok to fire here
+   //       }
+   //       else
+   //       {
+   //          _last_cached_state = dp.newRows;
+   //       }
+   //       _possibly_fire(dp.newRows);
+   //    }
+
+   //    public void doWork()
+   //    {
+   //       if (_unfired_dp != null)
+   //       {
+
+   //       }
+   //    }
+
+   //    private void _possibly_fire()
+   //    {
+   //       if (_unfired_dp != null)
+   //       {
+   //          if (DateTime.Now - _last_fire_time > _min_fire_period)
+   //          {
+   //             DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
+   //             _unfired_dp = null;
+   //             _last_Fire_time = DateTime.Now;
+   //             myFireEvent(dp);
+   //          }
+   //       }
+   //    }
+
+   // }
 
 
 }

src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs

@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Security.Cryptography.X509Certificates;
-
-using BCG;
-using BCG.Utilities.General;
-
-using HtmlAgilityPack;
-
-namespace BCG {
-    class Program {
-        static void Main() {
-            string url = "http://quotes.toscrape.com";
-            List<string> quotes = ScrapeQuotes(url);
-
-            Console.WriteLine("Quotes from http://quotes.toscrape.com:");
-            foreach (var quote in quotes) {
-                Console.WriteLine(quote);
-            }
-        }
-
-        static List<string> ScrapeQuotes(string url) {
-            List<string> quotes = new List<string>();
-
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-
-                // Extract quotes from the HTML document
-                var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
-                if (quoteNodes != null) {
-                    foreach (var quoteNode in quoteNodes) {
-                        string quote = quoteNode.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
-
-            return quotes;
-        }
-    }
-
-    public class WebScapperStream : ViewStream, IWorker {
-        private readonly string _url;
-        private readonly string _xpath;
-        private readonly ViewStreamDatabase _vsd;
-        private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
-        private readonly IBlueTableSchema _schema;
-
-        public WebScapperStream()
-        :
-        this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath)
-        : this(url, xpath, ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
-            _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("url", TypeCode.UInt32),
-                                                Tuple.Create("line", TypeCode.String));
-            _url = url;
-            _xpath = xpath;
-            _vsd = vsd;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            var lines = scrape_lines(_url, _xpath);
-            Func<string, int> get_id = (line) => {
-                int rv = 0;
-                if (!_line_id_map.TryGetValue(line, out rv))
-                    _line_id_map.Add(line, _line_id_map.Count);
-                return rv;
-            };
-            BlueTable nrows = new BlueTable(_schema);
-            foreach (var l in lines) {
-                bts.Log.Gen.info($"line: {l}");
-                var id = get_id(l);
-                BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
-                nrows.addRow(nr);
-            }
-            var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        static List<string> scrape_lines(string url, string xpath) {
-            List<string> quotes = new List<string>();
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-                // Extract quotes from the HTML document
-                var quote_nodes = document.DocumentNode.SelectNodes(xpath);
-                if (quote_nodes != null) {
-                    foreach (var qn in quote_nodes) {
-                        string quote = qn.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
-
-            return quotes;
-        }
-    }
-}