Reversehedging fixed

Previously, it was black and hard to read since the background is dark.

.config/dotnet-tools.json

@@ -9,7 +9,7 @@
       ]
     },
     "fable": {
-      "version": "4.0.6",
+      "version": "4.24.0",
       "commands": [
         "fable"
       ]
@@ -27,4 +27,4 @@
       ]
     }
   }
-}
+}

.devcontainer/Dockerfile

@@ -0,0 +1,131 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Note installations that take a while (more than 10s)
+
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
+
+# Python: ~30s
+RUN apt-get install -y \
+   python3-pip \
+   python3-venv \
+   && python3 -m pip install --upgrade pip
+
+# GCC
+ARG gcc_major=13
+RUN \
+   add-apt-repository ppa:ubuntu-toolchain-r/test \
+   && apt-get update \
+   && apt-get install -y g++-$gcc_major
+
+# Clang
+ARG clang_major=17
+RUN \
+   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+   | tee -a "/etc/apt/sources.list.d/llvm.list" \
+   && apt-get update \
+   && apt-get install -y \
+   clang-$clang_major \
+   clang-format-$clang_major \
+   clangd-$clang_major
+
+# CMake
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
+
+# Dotnet
+RUN apt-get update && apt-get install -y dotnet7
+
+# Node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+   | tee /etc/apt/sources.list.d/nodesource.list \
+   && apt-get update \
+   && apt-get install -y nodejs
+
+# Emumba (deps)
+RUN apt-get update && apt-get -y install rapidjson-dev
+# Emumba
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
+
+# Pcap
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+
+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
+# Set up default executables (update-alternatives)
+RUN \
+   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+
+# Add compiler symlinks for ccache 
+RUN \
+   ln -s /usr/bin/ccache /usr/local/bin/clang; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+   ln -s /usr/bin/ccache /usr/local/bin/g++
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+   groupadd -g "$docker_uid" "$docker_user" \
+   && useradd \
+   -m \
+   -u "$docker_uid" \
+   -g "$docker_uid" \
+   -G sudo \
+   --password "$(openssl passwd -1 "$docker_pass")" \
+   "$docker_user"

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+{
+  "dockerFile": "Dockerfile",
+  "hostRequirements": {
+    "memory": "8gb",
+    "storage": "10gb"
+  },
+  "containerUser": "bts",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "8gb"
+  ],
+  "mounts": [
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+  ],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "eamodio.gitlens",
+        "llvm-vs-code-extensions.vscode-clangd",
+        "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
+      ]
+    }
+  }
+}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.env

@@ -0,0 +1,48 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitattributes

@@ -0,0 +1,3 @@
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf

.gitignore

@@ -8,7 +8,9 @@ cmake-build-debug/
 .idea/
 .dir-locals.el
 *.pyc
-**.vscode
+**.vscode/*
+!.vscode/launch.json
+!.vscode/tasks.json
 .vs
 obj/
 *.csproj.user
@@ -54,6 +56,7 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-**/.devcontainer
+.devcontainer/
 *.Generated.cs
 *.lst
+.env.local

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.vscode/launch.json

@@ -0,0 +1,329 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": ".NET Core Attach",
+      "type": "coreclr",
+      "request": "attach"
+    },
+    {
+      "name": "DataStreamPublisher",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "run"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests2",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader2]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "DataStreamQueryAnalyzer",
+      "type": "coreclr",
+      "request": "launch",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
+        "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "PubSubServer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Parameterized)",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build Spark (Debug)",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "args": [
+          "--sparkd",
+          "${input:sparkConfigServer}"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net7.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net7.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sparkd",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+  ],
+  "inputs": [
+    {
+      "id": "dataStreamPublisherConfig",
+      "type": "pickString",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "sparkConfigServer",
+      "type": "promptString",
+      "description": "Config Server URL",
+      "default": "http://localhost:8000/config",
+    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,930 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Build Api Server",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Api Server (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Api Server (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--preset",
+          "${input:sparkdCmakePreset}",
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "${input:sparkdCompiler}"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug | Gcc)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "gcc"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug | Clang)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "clang"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "${input:sparkdCompiler}"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release | Gcc)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "gcc"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release | Clang)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "clang"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Edit DataStream Config",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "editor",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/conf/spark.conf"
+        ],
+      },
+      "windows": {
+        "command": "vim",
+        "args": [
+          "${env:APPDATA}/bts/DataStream/conf/spark.conf"
+        ],
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "Edit Spark Config",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "vim",
+        "args": [
+          "${env:APPDATA}/bts/conf/spark.conf"
+        ],
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "dedicated",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Spark",
+      "dependsOn": "Build Spark",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Release)",
+      "dependsOn": "Build Spark (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark Config Query",
+      "dependsOn": "Build Spark",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "Spark.Console",
+      },
+      "windows": {
+        "command": "Spark.Console.exe",
+      },
+      "args": [
+        "config",
+        "--query",
+        "${input:sparkConfigQuery}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd Config Query",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "config",
+        "--query",
+        "${input:sparkdConfigQuery}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Test Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+      "command": "",
+      "windows": {
+        "command": "Spark.Console.exe",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        }
+      },
+    },
+    {
+      "label": "Test Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
+    {
+      "label": "Test Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
+    {
+      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "DataStreamQueryAnalyzer.exe",
+          "'Starting DataStreamQueryAnalyzer'"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+  ],
+  "inputs": [
+    {
+      "id": "buildConfiguration",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "Debug",
+        "Release",
+      ],
+      "default": "Debug",
+    },
+    {
+      "id": "sparkdCmakePreset",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "sparkd-debug-clang",
+        "sparkd-debug-gcc",
+        "sparkd-release-clang",
+        "sparkd-release-gcc",
+      ],
+      "default": "sparkd-debug-clang",
+    },
+    {
+      "id": "sparkdCompiler",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "clang",
+        "gcc",
+      ],
+      "default": "sparkd-debug-clang",
+    },
+    {
+      "id": "dataStreamPublisherConfig",
+      "type": "pickString",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "sparkConfigQuery",
+      "type": "promptString",
+      "description": "Config Type(s)"
+    },
+    {
+      "id": "sparkdConfigQuery",
+      "type": "promptString",
+      "description": "Config Type(s)"
+    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+  ]
+}

CMakeLists.txt

@@ -1,7 +1,10 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)
 
+get_version_from_git()
 if(WIN32)
   set(platform "windows")
 else()
@@ -14,12 +17,17 @@ set(
   CACHE
   STRING
   "External dependencies root folder (should contain folders: lib, bin, include")
+
+# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
+# can run simultaneously without fighting over the same shm files (e.g., sparkd,
+# api server, etc.).
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
 set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
 set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
-set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
+set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
 if (WIN32)
   set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
   set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -31,29 +39,21 @@ endif()
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})
 
-if(BUILD_API_SERVER)
-  add_compile_definitions(SPARK_ENABLE_API_SERVER)
+add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
-
+add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
-  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
-  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
   # subdirs
   add_subdirectory(src/lib)
 else()
   # make sparkd look for libspark.so in its directory
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
   set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
-  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
-
-  if(BUILD_API_SERVER)
-    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
-    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
-  elseif(BUILD_SPARKD)
-    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
-    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
 
+  if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
     add_compile_definitions(SPARK_ENABLE_CCAPI)
@@ -90,9 +90,15 @@ else()
     add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
     # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
+    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
+
+    # ** Binance Coin Futures **
+    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
     # ** Deribit **
     add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -112,6 +118,7 @@ else()
 
   # subdirs
   if (BUILD_SPARKD)
+    add_subdirectory(src/lib_binance_execution)
     add_subdirectory(src/server/sparkd)
     add_subdirectory(src/data_files)
   endif()
@@ -133,6 +140,8 @@ else()
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
       COMMAND
         ln ARGS -sf
         ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -146,11 +155,14 @@ else()
 
     install(
       DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
     install(
       FILES
       "${SPARK_LIB}/libACE-5.7.5.so"
       "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
   endif()
 endif()

CMakePresets.json

@@ -15,22 +15,30 @@
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build",
+      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
         "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
       }
     },
 
@@ -46,13 +54,15 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "ON"
         },
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
@@ -62,7 +72,8 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
       }
     },
 
@@ -95,98 +106,72 @@
     {
       "name": "sparkd-debug-clang",
       "inherits": "sparkd-debug",
-      "displayName": "Debug (clang)",
+      "displayName": "Sparkd (Debug | Clang)",
       "description": "Debug build compiled w/ clang",
-      "binaryDir": "${sourceDir}/build/clang/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
       }
     },
 
     {
       "name": "sparkd-debug-gcc",
       "inherits": "sparkd-debug",
-      "displayName": "Debug (gcc)",
+      "displayName": "Sparkd (Debug | Gcc)",
       "description": "Debug build compiled w/ gcc",
-      "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
       }
     },
 
     {
       "name": "sparkd-release-clang",
       "inherits": "sparkd-release",
-      "displayName": "Release (clang)",
+      "displayName": "Sparkd (Release | Clang)",
       "description": "Release build compiled w/ clang",
-      "binaryDir": "${sourceDir}/build/clang/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
       }
     },
 
     {
       "name": "sparkd-release-gcc",
       "inherits": "sparkd-release",
-      "displayName": "Release (gcc)",
+      "displayName": "Sparkd (Release | Gcc)",
       "description": "Release build compiled w/ gcc",
-      "binaryDir": "${sourceDir}/build/gcc/release",
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
-      }
-    },
-
-    {
-      "name": "spark-api-server-base",
-      "hidden": true,
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
       }
     },
 
     {
       "name": "spark-api-server-windows",
       "inherits": "spark",
-      "displayName": "Spark Api Server",
+      "displayName": "Spark Api Server (Windows)",
       "cacheVariables": {
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        }
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
       }
     },
 
     {
       "name": "spark-api-server-linux",
       "inherits": "sparkd",
-      "displayName": "Spark Api Server",
+      "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
         },
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        },
         "BUILD_SPARKD": {
           "type": "BOOL",
           "value": "OFF"
@@ -197,7 +182,11 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
       }
     },
 
@@ -220,13 +209,62 @@
     },
 
     {
-      "name": "data-stream-suite",
+      "name": "data-stream-suite-windows",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
       "inherits": "spark",
-      "displayName": "Data Stream Suite",
+      "displayName": "Data Stream Suite (Windows)",
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
+    },
+
+    {
+      "name": "data-stream-suite-linux",
+      "displayName": "Data Stream Suite (Linux)",
+      "description": "Data Stream utils ported over from Edge",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "inherits": "sparkd",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
       }
     }
   ],
@@ -244,7 +282,7 @@
 
     {
       "name": "spark-debug",
-      "displayName": "Debug",
+      "displayName": "Spark (Debug)",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Debug"
@@ -252,7 +290,7 @@
 
     {
       "name": "spark-release",
-      "displayName": "Release",
+      "displayName": "Spark (Release)",
       "inherits": "spark",
       "configurePreset": "spark",
       "configuration": "Release"
@@ -265,43 +303,34 @@
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
-      "hidden": true,
-      "targets": [
-        "data_files_dir",
-        "admin_web_app_bundle",
-        "install",
-        "libspark",
-        "rapidjson_project",
-        "server_layout",
-        "sparkd"
-      ]
+      "hidden": true
     },
 
     {
       "name": "sparkd-debug-clang",
       "configurePreset": "sparkd-debug-clang",
-      "displayName": "Debug (clang)",
+      "displayName": "Sparkd (Debug | Clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-debug-gcc",
       "configurePreset": "sparkd-debug-gcc",
-      "displayName": "Debug (gcc)",
+      "displayName": "Sparkd (Debug | Gcc)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-clang",
       "configurePreset": "sparkd-release-clang",
-      "displayName": "Release (clang)",
+      "displayName": "Sparkd (Release | Clang)",
       "inherits": "sparkd"
     },
 
     {
       "name": "sparkd-release-gcc",
       "configurePreset": "sparkd-release-gcc",
-      "displayName": "Release (gcc)",
+      "displayName": "Sparkd (Release | Gcc)",
       "inherits": "sparkd"
     },
 
@@ -318,14 +347,14 @@
 
     {
       "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Windows | Debug)",
       "inherits": "spark-api-server-windows",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Windows | Release)",
       "inherits": "spark-api-server-windows",
       "configuration": "Release"
     },
@@ -344,14 +373,14 @@
     {
       "name": "spark-api-server-linux-debug",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
       "name": "spark-api-server-linux-release",
       "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Release)",
+      "displayName": "Spark Api Server (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -371,16 +400,30 @@
     },
 
     {
-      "name": "data-stream-suite-debug",
-      "displayName": "DataStreamSuite (Debug)",
-      "configurePreset": "data-stream-suite",
+      "name": "data-stream-suite-windows-debug",
+      "displayName": "Data Stream Suite (Windows | Debug)",
+      "configurePreset": "data-stream-suite-windows",
       "configuration": "Debug"
     },
 
     {
-      "name": "data-stream-suite-release",
-      "displayName": "DataStreamSuite (Release)",
-      "configurePreset": "data-stream-suite",
+      "name": "data-stream-suite-windows-release",
+      "displayName": "Data Stream Suite (Windows | Release)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "data-stream-suite-linux-debug",
+      "displayName": "Data Stream Suite (Linux | Debug)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-linux-release",
+      "displayName": "Data Stream Suite (Linux | Release)",
+      "configurePreset": "data-stream-suite-linux",
       "configuration": "Release"
     }
   ]

Spark.Api.Server.sln

@@ -14,6 +14,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +98,102 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -80,4 +204,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal

cmake/GitVersion.cmake

@@ -0,0 +1,48 @@
+include_guard(GLOBAL)
+
+function(get_version_from_git)
+  find_package(Git QUIET)
+  if(NOT Git_FOUND)
+    message(WARNING "Git not found")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --tags --always
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_TAG
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE GIT_RESULT
+  )
+
+  if(NOT GIT_RESULT EQUAL 0)
+    message(WARNING "Failed to get git tag")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
+
+  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
+  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
+
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
+  else()
+    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
+  endif()
+endfunction()

cmake/ProtobufGenerateFs.cmake

@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_fs)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
+  endif()
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  set(_outputs)
+  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
+    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
+    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
+    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
+    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
+  endforeach()
+
+  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  add_custom_command(
+    OUTPUT ${_outputs}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${_venv_exec}
+      ${_ARGS_PROTOC}
+      --plugin=protoc-gen-service=${_plugin_py}
+      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${_venv_exec}
+      ${_plugin_py}
+      ${_template_fs}
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
+endfunction()

cmake/ProtobufGenerateJs.cmake

@@ -0,0 +1,33 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_js)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUTPUT)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -p ${_dir})
+  endforeach()
+
+  add_custom_command(
+    OUTPUT ${_ARGS_OUTPUT}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx
+      --no-install pbjs
+      -t json-module
+      -w commonjs
+      -o ${_ARGS_OUTPUT}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
+endfunction()

cmake/SymLink.cmake

@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+
+function(sym_link_dir)
+  set(_options)
+  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
+    COMMENT "${_comment}"
+  )
+endfunction()
+
+function(sym_link_file)
+  set(_options)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
+  )
+  if (DEFINED _ARGS_INSTALL_DEST)
+    install(
+      FILES
+      ${_ARGS_SOURCE}
+      DESTINATION ${_ARGS_INSTALL_DEST}
+    )
+  endif()
+endfunction()
+
+function(sym_link_local_lib LIB_NAME)
+  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
+  file(
+    CREATE_LINK
+      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
+    SYMBOLIC
+  )
+endfunction()

cmake/TernaryOperator.cmake

@@ -0,0 +1,9 @@
+include_guard(GLOBAL)
+
+macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
+  if(${CONDITION})
+    set(${RESULT} ${TRUE_VALUE})
+  else()
+    set(${RESULT} ${FALSE_VALUE})
+  endif()
+endmacro()

cmake/version.h.in

@@ -0,0 +1,11 @@
+#ifndef BTS_SPARK_VERSION_H
+#define BTS_SPARK_VERSION_H
+
+#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
+#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
+#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
+#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
+#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
+#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
+
+#endif

compose.yaml

@@ -0,0 +1,100 @@
+services:
+  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

playground-1.mongodb.js

@@ -1,45 +0,0 @@
-/* global use, db */
-// MongoDB Playground
-// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
-// Make sure you are connected to enable completions and to be able to run a playground.
-// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
-// The result of the last command run in a playground is shown on the results panel.
-// By default the first 20 documents will be returned with a cursor.
-// Use 'console.log()' to print to the debug output.
-// For more documentation on playgrounds please refer to
-// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
-
-// Select the database to use.
-use('mongodbVSCodePlaygroundDB');
-
-// Insert a few documents into the sales collection.
-db.getCollection('sales').insertMany([
-    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
-    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
-]);
-
-// Run a find command to view items sold on April 4th, 2014.
-const salesOnApril4th = db.getCollection('sales').find({
-    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
-}).count();
-
-// Print a message to the output window.
-console.log(`${salesOnApril4th} sales occurred in 2014.`);
-
-// Here we run an aggregation and open a cursor to the results.
-// Use '.toArray()' to exhaust the cursor to return the whole result set.
-// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
-db.getCollection('sales').aggregate([
-    // Find all of the sales that occurred in 2014.
-    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
-    // Group the total sales for each product.
-    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
-]);
-
-

requirements.txt

@@ -1,11 +1,20 @@
 astroid==2.7.3
 autopep8==1.5.7
+blinker==1.9.0
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
+click==8.1.8
 colorama==0.4.4
+Flask==3.1.0
 idna==3.4
 isort==5.9.3
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
+MarkupSafe==3.0.2
 mccabe==0.6.1
 platformdirs==2.3.0
 protobuf==4.21.12
@@ -14,9 +23,14 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0

src/DataStreamSuite/BaseUtilities/Crono.cs

@@ -1,41 +1,22 @@
-using mbts.spark;
+using System.Diagnostics;
 
-using System;
-using System.Runtime;
+namespace BCG.Utilities.General;
 
-namespace BCG.Utilities.General
-{
-	/// <summary>
-	/// Summary description for Crono.
-	/// </summary>
-	public class Crono
-	{
-      public Crono()
-      {
-         myTimer = new HighPerformanceTimer();
-      }
+/// <summary>
+/// Summary description for Crono.
+/// </summary>
+public class Crono {
+   readonly Stopwatch myTimer = new();
 
-      public void start() 
-      {myTimer.start();}
+   public void start() =>
+      myTimer.Start();
 
-      public void stop()
-      {myTimer.stop();}
+   public void stop() =>
+      myTimer.Stop();
 
-      public TimeSpan time
-      {
-         get 
-         {
-            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
-         }
-      }
-
-      public double miliseconds
-      {
-         get
-         {return myTimer.totalMilliseconds;}
-      }
-
-      private HighPerformanceTimer myTimer;
-	}
+   public TimeSpan time =>
+      myTimer.Elapsed;
 
+   public double miliseconds =>
+      myTimer.Elapsed.TotalMilliseconds;
 }

src/DataStreamSuite/BlueRegistry/BlueRegistry.cs

@@ -1,192 +1,192 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Reflection;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public enum RegistryRoot { HKCU, HKLM }
-
-   [Serializable]
-   public class BlueRegistry
-   {
-      public const string BlueSubKey = @"Software\Blue Capital Group\";
-      public static readonly string ApplicationPrefix;
-      protected string myFullPath;
-
-      public BlueRegistry(string subPath)
-      {
-         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
-      }
-
-      public BlueRegistry(BlueRegistry parent, string subPath)
-      {
-         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
-      }
-
-      static BlueRegistry()
-      {
-         string app = "Applications";
-
-         Assembly ass = Assembly.GetEntryAssembly();
-         if (ass != null)
-         {
-            app = ass.GetName().Name;
-
-            object[] attrs = ass.GetCustomAttributes(
-               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
-
-            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
-            {
-               app = ((AssemblyTitleAttribute)attrs[0]).Title;
-            }
-         }
-
-         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
-      }
-
-      public object getValue(string name)
-      {
-         return getValue(RegistryRoot.HKCU, name);
-      }
-
-      public object getValue(string name, object defaultValue)
-      {
-         return getValue(RegistryRoot.HKCU, name, defaultValue);
-      }
-
-      public void setValue(string name, object value)
-      {
-         setValue(RegistryRoot.HKCU, name, value);
-      }
-
-      public void deleteValue(string name)
-      {
-         deleteValue(RegistryRoot.HKCU, name);
-      }
-
-      private RegistryKey myGetRegistryKey(RegistryRoot root)
-      {
-         if (root == RegistryRoot.HKCU)
-            return Registry.CurrentUser.CreateSubKey(myFullPath);
-         else
-            return Registry.LocalMachine.CreateSubKey(myFullPath);
-      }
-
-      #region Generic Methods
-      internal virtual void setValue(RegistryRoot root, string name, object value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.SetValue(name, value);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name);
-         }
-      }
-
-      internal virtual void deleteValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.DeleteValue(name);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name, defaultValue);
-         }
-      }
-      #endregion
-
-      #region Double
-      internal virtual void setDouble(RegistryRoot root, string name, double value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            DoubleRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual double getDouble(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return DoubleRegistryHandler.getValue(rk, name);
-         }
-      }
-      #endregion
-
-      #region Enum
-      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            EnumRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return EnumRegistryHandler.getValue(rk, name, enumType);
-         }
-      }
-      #endregion
-
-      #region Array
-      internal virtual void setArray(RegistryRoot root, string name, Array array)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            ArrayRegistryHandler.setValue(rk, name, array);
-         }
-      }
-
-      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return ArrayRegistryHandler.getValue(rk, name, elementType);
-         }
-      }
-      #endregion
-
-      #region SortedList
-      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.setValue(rk, name, slist);
-         }
-      }
-
-      internal virtual SortedList getSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return SortedListRegistryHandler.getValue(rk, name);
-         }
-      }
-
-      internal virtual void deleteSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.deleteValue(rk, name);
-         }
-      }
-      #endregion
-   }
-}
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      public virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      public virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,10 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-    <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>
 
 </Project>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -18,8 +18,11 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -0,0 +1,26 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Reactive.Linq;
+
+public static class DataStreamExtensions {
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
+      source as ViewStream ?? new ObservableViewStream(source);
+
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
+         source.onChange += OnChange;
+         return Disposable.Create(() => source.onChange -= OnChange);
+
+         void OnChange(object? newValue) =>
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
+      }
+   }
+}

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -0,0 +1,32 @@
+namespace bts.SilverPine {
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream {
+      private readonly string _dsl;
+      public DSLStream(string dsl) {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         _dsl = dsl;
+         ViewStream rs = build_stream(dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(string stream_expression) {
+         try {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         } catch (Exception ex) {
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Misc/PandasScriptStream.cs

@@ -0,0 +1,29 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+using System.IO;
+
+public class PandasScriptStream : ViewStream {
+   public PandasScriptStream(ViewStream vs, string file)
+      : this(vs, file, null) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
+      : this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
+      mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
+
+      string readFile() {
+         try {
+            return File.ReadAllText(file);
+         } catch (Exception ex) {
+            Log.Gen.error($"PandasScriptStream error reading file: {ex}");
+            throw;
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,62 +1,130 @@
-// using System;
-// using System.Diagnostics;
-
-// using BCG;
-// using BCG.Utilities.General;
-
-
-// namespace bts {
-
-//     public class PandasStream : ViewStream, IWorker {
-//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
-
-//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
-
-//         }
-
-//         public PandasStream(ViewStream vs, string pandas_code) :
-//         this(vs, ViewStreamDatabase.instance, pandas_code) {
-//         }
-
-
-//         protected override void myOnFirstSubscriber() {
-
-//             // string python_code = @"
-//             //    import winsound
-//             //    frequency = 2500  # Set frequency (in Hz)
-//             //    duration = 1000   # Set duration (in milliseconds)
-//             //    winsound.Beep(frequency, duration)
-//             // ";
-//             string python_code = @"
-//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
-//             with open(file_path, \"w\") as f:
-//                 f.write(\"Hello, world!\n\")
-//             ";
-
-//             execute_python_code(python_code);
-//         }
-
-//         static void execute_python_code(string python_code) {
-//             ProcessStartInfo start = new ProcessStartInfo();
-//             start.FileName = "python"; // Python executable
-//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
-//             start.UseShellExecute = false;
-//             start.RedirectStandardOutput = true;
-
-//             using (Process process = Process.Start(start)) {
-//                 using (System.IO.StreamReader reader = process.StandardOutput) {
-//                     string result = reader.ReadToEnd();
-//                     Console.Write(result);
-//                 }
-//             }
-//         }
-
-//         protected override void myOnLastSubscriber() {
-//         }
-
-//         public void doWork() {
-
-//         }
-
-//     }
-// }
+namespace Bts.DataStream;
+
+using BCG;
+using BCG.Extensions;
+using BCG.Utilities.General;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+public class PandasStream : ViewStream, IWorker {
+   readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
+   readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
+   readonly string pythonExe;
+   readonly ViewStreamDatabase vsd;
+   readonly CSVSaverStream inputCsvStream;
+   readonly Lazy<string> pythonCode;
+   IDictionary<int, IBlueRow>? lastRowMap = null;
+   int inputVersion = 0;
+   int outputVersion = 0;
+   DateTime lastFireTime = DateTime.MinValue;
+   TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
+   IDisposable? dispose;
+
+   public PandasStream(ViewStream vs, string dataFrameExpression)
+      : this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
+
+   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
+      pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
+      this.pythonExe = pythonExe;
+      this.vsd = vsd;
+      // This should fire an update when it writes to the output file
+      inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
+      pythonCode = new(getPythonCode);
+
+      string getPythonCode() {
+         var imports =
+@"import numpy as np
+import pandas as pd
+import sys
+";
+         var bd = new StringBuilder(imports);
+         bd.AppendLine($"df=pd.read_csv('{inputFile}')");
+         bd.AppendLine(dataFrameExpression);
+         bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
+         return bd.ToString();
+      }
+   }
+
+   protected override void myOnFirstSubscriber() {
+      vsd.addWorker(this);
+      inputCsvStream.onChange += onInputChanged;
+      this.dispose = Disposable.Create(dispose);
+
+      void onInputChanged(object newValue) =>
+         inputVersion++;
+
+      void dispose() {
+         inputCsvStream.onChange -= onInputChanged;
+         vsd.removeWorker(this);
+         deleteFile(inputFile);
+         deleteFile(outputFile);
+      }
+
+      void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
+         if (!File.Exists(file)) {
+            return;
+         }
+         try {
+            File.Delete(file);
+         } catch (Exception ex) {
+            Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref dispose);
+
+   static void executePythonCode(string pythonCode, string? pythonExe = null) {
+      pythonExe ??= "python";
+      var start = new ProcessStartInfo {
+         FileName = pythonExe,
+         Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
+         UseShellExecute = false,
+         RedirectStandardOutput = true
+      };
+      using var process = Process.Start(start)
+         ?? throw new InvalidOperationException("Failed to launch python");
+      process.WaitForExit();
+   }
+
+   void IWorker.doWork() {
+      if (outputVersion == inputVersion) {
+         return;
+      }
+      var now = DateTime.Now;
+      var elapsedSinceLastFire = now - lastFireTime;
+      if (elapsedSinceLastFire < fireThrottle) {
+         return;
+      }
+      outputVersion = inputVersion;
+      executePythonCode(pythonCode.Value, pythonExe);
+      var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
+      var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
+      var comp = BlueRowEqualityComparer.Instance;
+      var deletedRows = new BlueTable(schema);
+      var addedRows = new BlueTable(schema);
+      if (lastRowMap is not null) {
+         deletedRows.SetItems(lastRowMap.Where(oldRow =>
+            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
+         addedRows.SetItems(rowMap.Where(newRow =>
+            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
+      }
+      lastRowMap = rowMap;
+      lastFireTime = DateTime.Now;
+      myFireEvent(new DataPacket(rows, addedRows, deletedRows));
+   }
+}

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -39,8 +39,8 @@ namespace BCG {
 
         protected override void myOnFirstSubscriber() {
             MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_address = new BlutoSubAddress(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
+            var server_rci = new RemoteConnectionInfo(_ip, _port);
+            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
         }
 
         protected override void myOnLastSubscriber() {

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -0,0 +1,37 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
+   IDisposable? subscription;
+
+   public ObservableViewStream(IObservable<DataPacket> source) =>
+      this.source = source;
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      if (subscription is not null) {
+         subscription.Dispose();
+         subscription = null;
+      }
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(subscription);
+      subscription = source.Subscribe(myFireEvent);
+   }
+
+   protected override void myOnLastSubscriber() {
+      Log.notnull(subscription);
+      subscription.Dispose();
+      subscription = null;
+   }
+
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
+      source.Subscribe(observer);
+}

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -1,32 +0,0 @@
-using BCG;
-
-namespace BCG
-{
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("add_to_cloud")]
-   public class AddToCloudStream : ViewStream
-   {
-      private readonly string _name, _stream_exp;
-      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
-      public AddToCloudStream(string name, params string[] lines)
-      : this(name, 0, lines)
-      {
-      }
-
-      public AddToCloudStream(string name, int persist, params string[] lines)
-      {
-         _name = name;
-         _stream_exp = getExpression(lines);
-         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
-         mySetRealSource(new TableStream("cloud_streams"));
-      }
-
-      static string getExpression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,31 +1,22 @@
-using BCG;
-
 using System;
 using System.Collections.Generic;
 
-using mbts.config;
-using bts.utils;
-using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
-using System.Collections.Generic;
 using BCG.Utilities.General;
+using bts;
 
-namespace BCG
-{
+namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class CloudStreamManager : IWorker
-   {
-      public class StreamPubUnit
-      {
+   public class RemoteQueryManager : IWorker {
+      public class StreamPubUnit {
          public string name;
          public string stream_expression;
          public ViewStream stream;
          public bool persist;
          public ViewPublisher view_publisher;
 
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
-         {
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
             name = name_;
             stream_expression = stream_expression_;
             stream = stream_;
@@ -33,7 +24,7 @@ namespace BCG
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream build_stream(string stream_expression)
+      static ViewStream? build_stream(string stream_expression)
       {
          try
          {
@@ -41,9 +32,7 @@ namespace BCG
             var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
             bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
@@ -52,34 +41,37 @@ namespace BCG
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       private ViewStream _stream = new TableStream("cloud_streams");
       private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
-      public CloudStreamManager()
-      {
+
+      public RemoteQueryManager() {
          _stream.onChange += _on_str;
-
+         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
       }
       private void _on_str(object obj)
       {
-         DataPacket dp = obj as DataPacket;
+         if (obj is not DataPacket dp) {
+            return;
+         }
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
             bool persist = r.get<bool>("persist");
             string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name))
-            {
-               _assign(name, stream_exp, persist);
+            if (!_stream_pub_map.ContainsKey(name)) {
+               _spawn_remote_stream(name, stream_exp, persist);
             }
          }
       }
 
-      private void _assign(string name, string stream_expression, bool persist)
-      {
+      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
-         if (persist)
-         {
+         if (persist) {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -87,14 +79,12 @@ namespace BCG
          _stream_pub_map.Add(name, u);
       }
 
-      private void _expose_to_ws(string name, ViewStream stream)
-      {
+      private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
 
-      public void doWork()
-      {
+      public void doWork() {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -0,0 +1,48 @@
+using BCG;
+
+using bts;
+
+using System;
+
+namespace BCG {
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
+   public class RemoteQueryStream : ViewStream {
+      private readonly string _name, _stream_exp;
+      private PublisherCore _publisher_core = new PublisherCore();
+      private ViewStream _source_str;
+
+      public RemoteQueryStream(params string[] lines) {
+         _stream_exp = get_expression(lines);
+         _name = Guid.NewGuid().ToString();
+         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
+         //_name.Replace("-", "_");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
+         byte[] rem_req_bytes = rem_req.serialize();
+         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
+      }
+
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }
+
+      static string get_expression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+
+      private void _handle_data(object o) {
+         myFireEvent(o);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -0,0 +1,31 @@
+namespace bts {
+   using System.IO;
+   using BCG;
+
+   using System.IO;
+   using System.Threading.Tasks.Dataflow;
+
+   public class RemoteRequest {
+      public readonly string stream_expression;
+      public readonly string published_table_name;
+      public RemoteRequest(string stream_expression_, string published_table_name_) {
+         stream_expression = stream_expression_;
+         published_table_name = published_table_name_;
+      }
+      public byte[] serialize() {
+         var str = new MemoryStream();
+         var bw = new BinaryWriter(str);
+         bw.write_string_cpp(stream_expression);
+         bw.write_string_cpp(published_table_name);
+         return str.ToArray();
+      }
+
+      public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
+         BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
+         string stream_expression_ = br.read_string_cpp();
+         string published_table_name_ = br.read_string_cpp();
+         return new RemoteRequest(stream_expression_, published_table_name_);
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -1,59 +1,54 @@
-using BCG;
+// using BCG;
 
-using System.Linq;
+// using bts;
 
-namespace BCG
-{
-   public class StreamOffloadRequestor
-   {
-      public class OffloadOp
-      {
-         string name;
-         string stream_exp;
-      }
-      private IBlueTableSchema _schema;
-      private IBlueRowCollection _rows;
-      private PubTableBroker _broker = new PubTableBroker();
-      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-      public StreamOffloadRequestor()
-      {
-         _schema = new BlueTableSchema("cloud_streams",
-                                   new ColumnInfo("id", 0, typeof(int)),
-                                   new ColumnInfo("name", 1, typeof(string)),
-                                   new ColumnInfo("persist", 2, typeof(bool)),
-                                   new ColumnInfo("stream_exp", 3, typeof(string)));
-         _existing_offloads_vs.onChange += _on_str;
-      }
-      private void _on_str(object obj)
-      {
-         DataPacket dp = obj as DataPacket;
-         _rows = dp.newRows;
-         // this is needed so the broker knows the state of the table
-         _broker.createTable(_rows);
-      }
-      public void add_to_cloud(string name, string stream_exp, bool persist)
-      {
-         // check if it exists
-         if (_rows == null)
-         {
-            BlueTable table = new BlueTable(_schema);
-            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-            table.addRow(nr);
-            _broker.createTable(table);
-            // create table  
-         }
-         else
-         {
-            if (exists(name))
-               return;
-            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-            _broker.addRow(nr);
-         }
-      }
+// using System.Linq;
 
-      bool exists(string name)
-      {
-         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-      }
-   }
-}
+// namespace BCG {
+//    // used on the client
+//    public class StreamOffloadRequestor {
+//       public class OffloadOp {
+//          string name;
+//          string stream_exp;
+//       }
+//       private IBlueTableSchema _schema;
+//       private IBlueRowCollection _rows;
+//       private PubTableBroker _broker = new PubTableBroker();
+//       ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+//       public StreamOffloadRequestor() {
+//          _schema = new BlueTableSchema("cloud_streams",
+//                                    new ColumnInfo("id", 0, typeof(int)),
+//                                    new ColumnInfo("name", 1, typeof(string)),
+//                                    new ColumnInfo("persist", 2, typeof(bool)),
+//                                    new ColumnInfo("stream_exp", 3, typeof(string)));
+//          _existing_offloads_vs.onChange += _on_str;
+//       }
+//       private void _on_str(object obj) {
+//          DataPacket dp = obj as DataPacket;
+//          _rows = dp.newRows;
+//          // this is needed so the broker knows the state of the table
+//          _broker.create_table(_rows);
+//       }
+//       public void add_to_cloud(string name, string stream_exp, bool persist) {
+//          // check if it exists
+//          if (_rows == null) {
+//             BlueTable table = new BlueTable(_schema);
+//             IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+//             table.addRow(nr);
+//             _broker.create_table(table);
+//             // create table  
+//          } else {
+//             if (exists(name))
+//                return;
+//             IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+//             // TODO: use make_batch_changes instead
+//             Log.assert(false, "Fix addRow");
+//             //_broker.addRow(nr);
+//          }
+//       }
+
+//       bool exists(string name) {
+//          return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs

@@ -1,17 +1,14 @@
 using System;
 using System.Collections;
 
-namespace BCG
-{
-   internal class AddColumnsRow : IBlueRow
-   {
+namespace BCG {
+   internal class AddColumnsRow : IBlueRow {
       private IBlueTableSchema mySchema;
       private IBlueRow myRow;
       //private object[] myCachedValues;
 
       #region Constructors
-      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
-      {
+      public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
          mySchema = schema;
          myRow = row;
          //myCachedValues = new object[schema.columnNames.Length];
@@ -19,85 +16,78 @@ namespace BCG
       #endregion
 
       #region IBlueRow members
-      public IBlueTableSchema schema{get{return mySchema;}}
+      public IBlueTableSchema schema { get { return mySchema; } }
       #endregion
 
       #region ICollection members
-      public int Count {get{return mySchema.columnNames.Length;}}
+      public int Count { get { return mySchema.columnNames.Length; } }
 
-      public bool IsSynchronized {get{return false;}}
+      public bool IsSynchronized { get { return false; } }
 
-      public object SyncRoot {get{return this;}}
+      public object SyncRoot { get { return this; } }
 
-      public void CopyTo (Array array, int index)  
-      {
+      public void CopyTo(Array array, int index) {
          int i = index;
          foreach (object o in this as IEnumerable)
-            array.SetValue (o, i++);
+            array.SetValue(o, i++);
       }
       #endregion
 
       #region IDictionary members
-      public bool IsFixedSize {get{return true;}}
+      public bool IsFixedSize { get { return true; } }
 
-      public bool IsReadOnly {get{return true;}}
+      public bool IsReadOnly { get { return true; } }
 
       /// <summary>
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public object this [object key]
-      {
-         get
-         {
+      public object this[object key] {
+         get {
             if (key is string)
                return this[(string)key];
             return this[(int)key];
          }
-         set {throw new NotSupportedException ("The IBlueRow is readonly");}
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
       }
 
-      public object this[int index]
-      {
-         get
-         {
+      public object this[int index] {
+         get {
             //if (myCachedValues[index] == null)
             //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
             //return myCachedValues[index];
-            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+            return mySchema[index].isConstant ? DBNull.Value : myRow[index];
          }
       }
 
-      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}
+      public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
 
-      public ICollection Keys{get{return mySchema.columnNames;}}
+      public ICollection Keys { get { return mySchema.columnNames; } }
 
-      public ICollection Values
-      {
-         get
-         {
+      public ICollection Values {
+         get {
             object[] retVal = new object[mySchema.columnNames.Length];
-            CopyTo (retVal, 0);
+            CopyTo(retVal, 0);
             return retVal;
          }
       }
 
-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
 
-      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
 
-      public bool Contains (object key){return mySchema[(string)key] != null;}
+      public bool Contains(object key) { return mySchema[(string)key] != null; }
 
       /// <summary>
       /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
       /// iterate through the values, typecast this into an IEnumerable
       /// </summary>
       /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
-      
-      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
 
-      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
       #endregion
 
    }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,6 +1,34 @@
-namespace BCG;
-
-public class AddColumnsStream : MapStream {
-   public AddColumnsStream(ViewStream vs, string columnDef)
-      : base(vs, new ColumnAdder(columnDef)) { }
-}
+using System;
+
+namespace BCG {
+   public class AddColumnsStream : ViewStream {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
+         string[] toks = exp.Split(':');
+         string col_name = toks[0];
+         string col_value = toks[1];
+         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
+         return Tuple.Create(col_name, tup.Item1, tup.Item2);
+      }
+
+      private Tuple<string, Type, object> _new_col;
+      private IBlueTableSchema _out_schema;
+
+      public AddColumnsStream(ViewStream vs, string col_def) {
+         _new_col = parse_exp(col_def);
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
+         mySetRealSource(rs);
+      }
+
+      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
+         return _out_schema;
+      }
+
+      private IBlueRow _map(IBlueRow r) {
+         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
+         rv.setValue(_new_col.Item1, _new_col.Item3);
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs

@@ -1,23 +1,24 @@
+using bts;
+
 using System;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation ("distinct")]
-	public class DistinctRowStream : ViewStream
-	{
-		public DistinctRowStream (ViewStream stream, params string[] cols)
-		{
-         ArrayList groupByCols = new ArrayList ();
-         foreach (string col in cols)
-            foreach (string c in col.Split (' '))
-               groupByCols.Add (c);
-         mySetRealSource (new GroupByStream (stream, 
-            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
+	[DataStreamUserRespresentation("distinct")]
+	public class DistinctRowStream : ViewStream {
+		public DistinctRowStream(ViewStream stream, params string[] cols) {
+			var g_by_eval = new DistinctRowEvaluator(cols);
+			ViewStream r = new GroupByStream(stream, g_by_eval);
+			var proj_exp = String.Empty;
+			foreach (var c in cols)
+				proj_exp += $"{c} ";
+			proj_exp = proj_exp.Trim();
+			r = new NewProjectStream(r, proj_exp);
+			mySetRealSource(r);
 		}
 	}
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,48 +1,25 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
+using BCG;
 
-using System.Runtime.Serialization.Formatters.Binary;
-
-namespace BCG
-{
+namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
    [DataStreamUserRespresentation("loader")]
-   public class LoaderStream : ViewStream, ISideEffectStream
-   {
-      public class Exception : System.Exception
-      {public Exception(string msg) : base(msg) { }}
+   public class LoaderStream : ViewStream {
 
-      public LoaderStream(string fileName, int secondsBetweenPolls)
-      {
-         mySetRealSource(
-            new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-               ViewStreamDatabase.instance));
+      private string _filename;
+      private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
+
+
+      public LoaderStream(string filename) {
+         _filename = filename;
       }
 
-      public LoaderStream(string fileName)
-      {
-        if (fileName == null)
-           throw new LoaderStream.Exception("File name cannot be null");
-
-        for (int i = 0; i < 5; i++)
-           try
-           {
-              using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-              {
-                  BinaryFormatter bf = new BinaryFormatter();
-#pragma warning disable SYSLIB0011
-                  BlueTable bt = (BlueTable)bf.Deserialize(fs);
-#pragma warning restore SYSLIB0011
-                  mySetRealSource(new BlueRowCollectionStaticStream(bt));
-              }
-              break;
-           }
-           catch (IOException)
-           {
-              System.Threading.Thread.Sleep(50);
-           }
-
+      protected override void myOnFirstSubscriber() {
+         byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
+         StreamIOMessage msg = _deser.deserialize(ser_msg);
+         Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
+         IBlueRowCollection rows = msg.create_message.rows;
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,9 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
+using BCG;
 
-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -24,7 +26,7 @@ namespace BCG {
       private ViewStream _source_stream;
       private string _token;
       private FileStream _out_fs;
-      private IBlueDataSerializer _serializer;
+      private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
       private string _file_name = String.Empty;
       private long _rollover_size;
       private readonly string _directory_path;
@@ -93,20 +95,19 @@ namespace BCG {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
-         if (_write_create) {
-            // new files always starts with create
-            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
+         _serializer = _serializer ?? new UniversalStreamIOSerializer();
+         if (first_write) {
+            // first file always starts with create
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
-            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
-
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
          BinaryWriter bw = new BinaryWriter(_out_fs);
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-
-namespace BCG
-{
-   public class RecorderStream : ViewStream
-   {
-      private ViewStream mySource;
-      private DateTime myStartTime;
-      private FileStream myOutFile;
-      private string myFileName;
-      private IBlueDataSerializer mySerializer;
-            
-      public RecorderStream(ViewStream src, string recordFile)
-      {
-         mySource = src;
-         myFileName = recordFile;
-
-         myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-      }
-
-      protected override void myOnFirstSubscriber()
-      {
-         myStartTime = DateTime.Now;         
-         mySource.onChange += new ChangeDelegate(mySource_onChange);
-         myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-      }
-      
-      protected override void myOnLastSubscriber()
-      {
-         mySource.onChange -= mySource_onChange;
-         myOutFile.Close();
-      }
-
-      void mySource_onChange(object newValue)
-      {
-         DataPacket dp = (DataPacket)newValue;         
-         
-         BlueTable toWrite = new BlueTable(dp.schema);
-
-         if (mySerializer == null)
-         {
-            mySerializer = new UniversalSerializer(dp.schema);
-         }
-         foreach (IBlueRow r in dp.newRows)
-            toWrite.addRow(r);
-         
-         byte[] result;
-
-         if (myLastValue == null)
-            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-         else
-            mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-         BinaryWriter bw = new BinaryWriter(myOutFile);
-         //write down what time this happened
-         bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-         bw.Write(result.Length);
-         bw.Write(result);         
-         myOutFile.Flush();
-
-         myFireEvent(dp);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -1,64 +1,37 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using BCG;
 
-// using System.Runtime.Serialization.Formatters.Binary;
+namespace bts {
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
+   [DataStreamUserRespresentation("saver")]
+   public class SaverStream : ViewStream {
+      private ViewStream _stream;
+      private string _filename;
+      private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
 
-// namespace BCG
-// {
-//    [DataStreamUserRespresentation("saver")]
-//    public class SaverStream : ViewStream, ISideEffectStream
-//    {
-//       private ViewStream myStream;
-//       private string myFileName;
+      public SaverStream(ViewStream vs, string filename) {
+         _stream = vs;
+         _filename = filename;
+      }
 
-//       public class Exception : System.Exception
-//       {
-//          public Exception(string msg) : base(msg) { }
-//       }
+      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
 
-//       public SaverStream(ViewStream vs, string fileName)
-//       {
-//          if (fileName == null)
-//             throw new SaverStream.Exception("File name cannot be null");
-//          myStream = vs;
-//          myFileName = fileName;
+      void _handle_changes(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _save(dp);
+         myFireEvent(dp_obj);
+      }
 
-//       }
+      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
 
-//       protected override void myOnFirstSubscriber()
-//       { myStream.onChange += new ChangeDelegate(myHandleChanges); }
-
-//       void myHandleChanges(object newValue)
-//       {
-//          DataPacket dp = newValue as DataPacket;
-//          mySave(dp);
-//          myFireEvent(newValue);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
-
-//       protected void mySave(DataPacket dp)
-//       {
-//          for (int i = 0; i < 5 ; i++)
-//             try
-//             {
-//                using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
-//                {
-//                   BinaryFormatter bf = new BinaryFormatter();
-//                   BlueTable t = new BlueTable(dp.schema);
-//                   foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
-//                      t.addRow(new BlueRow(r));
-//                   bf.Serialize(fs, t);
-//                }
-//                break;
-//             }
-//             catch (IOException)
-//             {
-//                System.Threading.Thread.Sleep(5);
-//             }
-//       }
-//    }
-// }
+      protected void _save(DataPacket dp) {
+         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
+         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
+         StreamSerializationUtils.write_to_file(str_bytes, _filename);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -1,12 +1,12 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using System.Runtime.Serialization.Formatters.Binary;
-using BCG.Utilities.General;
-using bts.collections;
-using System.Reflection.Metadata;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
 
 
 namespace BCG {
@@ -15,115 +15,115 @@ namespace BCG {
          FirstOfDay,
          LastOfDay
       };
-      private BlueTable _current_table;
-      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-      public DateTime _snapshot_time;
-      public string _snapshot_file;
-      public IEnumerable<PlaybackPacket> _packets;
-      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-         DateTime dt;
-         var parsed = DateTime.TryParse(snapshot_date, out dt);
-         dt = dt.Date;
-         bts.Log.assert(parsed);
-         SpecificSnapshot specific_snapshot_enum;
-         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-         bts.Log.assert(parsed);
-         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-         mySetRealSource(rs);
-      }
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       public DateTime _snapshot_time;
+//       public string _snapshot_file;
+//       public IEnumerable<PlaybackPacket> _packets;
+//       public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+//          DateTime dt;
+//          var parsed = DateTime.TryParse(snapshot_date, out dt);
+//          dt = dt.Date;
+//          bts.Log.assert(parsed);
+//          SpecificSnapshot specific_snapshot_enum;
+//          parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+//          bts.Log.assert(parsed);
+//          string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+//          ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+//          mySetRealSource(rs);
+//       }
 
-      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-         string token_directory_name = $"{directory_name}\\{token}";
-         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-      }
+//       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+//          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+//       }
 
-      protected override void myOnFirstSubscriber() {
-         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-         _packets = StreamPersistorUtils.load_packets_from_file(fs);
-         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-         if (snapshot_packet != null) {
-            foreach (PlaybackPacket p in _packets) {
-               byte[] ops;
-               Array row_array;
-               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-               for (int i = 0; i < ops.Length; i++) {
-                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-                  byte op = ops[i];
-                  if (op == StreamOpCode.create) {
-                     IBlueRowCollection full_table = rows_args[0];
-                     _current_table = new BlueTable(full_table);
-                  } else if (op == StreamOpCode.multipleRemoveAdd) {
-                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
-                  } else {
-                     bts.Log.assert(false, $"Oprator: {op} is not expected");
-                  }
-                  if (p.idx == snapshot_packet.idx) {
-                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-                     break;
-                  }
-               }
-            }
-         }
-      }
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+//          _packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
 
-      private void _absorb_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-            } else {
-               bts.Log.assert(false, $"Operator: {op} is not expected");
-            }
-         }
-      }
+//       private void _absorb_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//             } else {
+//                bts.Log.assert(false, $"Operator: {op} is not expected");
+//             }
+//          }
+//       }
 
-      protected override void myOnLastSubscriber() {
-      }
+//       protected override void myOnLastSubscriber() {
+//       }
 
-      // public event SnapshotStream.Progress onProgress;
-      // public event Action onFileExhausted;
-      private PlaybackPacket _locate_packet_to_replay() {
-         foreach (var p in _packets) {
-            // returns the first non replayed packet
-            if (!p.replayed)
-               return p;
-         }
-         return null;
-      }
+//       // public event SnapshotStream.Progress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          return null;
+//       }
 
-      private void _fire_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-               //_current_table.addRows(full_table);
-               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-                                          (myLastValue as DataPacket).newRows));
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-               myFireEvent(new DataPacket(_current_table, adds, dels));
-            } else {
-               bts.Log.assert(false, $"Oprator: {op} is not expected");
-            }
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
 
-         }
-      }
+//          }
+//       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;
 
 
 namespace BCG {

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,16 +1,10 @@
-using BCG;
-using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using System.Text.RegularExpressions;
-
 using Bts.DataStream;
 
-using mbts;
-using mbts.spark;
 using mbts.config;
 
+using System;
+using System.Text.RegularExpressions;
+
 namespace BCG.StreamWorld.Addition
 {
    [DataStreamUserRespresentation (false)]
@@ -151,7 +145,6 @@ namespace BCG
       private string myFilterString;
       private BCG.StreamWorld.Addition.FilterStream myFilterStream;
       private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -226,7 +219,6 @@ namespace BCG
 
       private void myHandleCallbacks(object newValue)
       {
-         _timer.start();
          DataPacket retVal;
          DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
@@ -255,7 +247,6 @@ namespace BCG
                if (myFilter.filter(r))
                   myLastFiredRows.addRow(r);
          }
-         _timer.stop();
          myFireEvent(retVal);
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs

@@ -0,0 +1,24 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name)
+      : this(parent_str, col_name, true) { }
+      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
+         ViewStream rs = new FilterStream(parent_str,
+         r => {
+            bool is_dbnull = r.is_null(col_name);
+            if (is_dbnull) {
+               return false;
+            }
+            if (r.schema[col_name].type_code() == TypeCode.Double) {
+               return !Double.IsNaN(r.get<double>(col_name));
+            }
+            return true;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -1,37 +1,66 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using System.Text.RegularExpressions;
 
 using BCG.Utilities.General;
 
-namespace BCG
-{
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
-   {
+namespace BCG {
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
+
+      public static TimeSpan parse_duration(string input) {
+         int days = 0, hours = 0, minutes = 0, seconds = 0;
+
+         // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
+         Regex regex = new Regex(@"(\d+)([dhms])");
+         MatchCollection matches = regex.Matches(input);
+
+         foreach (Match match in matches) {
+            int value = int.Parse(match.Groups[1].Value);
+            string unit = match.Groups[2].Value;
+
+            switch (unit) {
+               case "d":
+                  days = value;
+                  break;
+               case "h":
+                  hours = value;
+                  break;
+               case "m":
+                  minutes = value;
+                  break;
+               case "s":
+                  seconds = value;
+                  break;
+            }
+         }
+
+         return new TimeSpan(days, hours, minutes, seconds);
+      }
+
       #region constructors
+
+      public TimeBasedFilter(string col_name, string row_time_to_live)
+      : this(col_name, parse_duration(row_time_to_live)) {
+
+      }
+
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-      {
-         myVsd               = vsd;
-         myRowTimeColumnName = rowTimeColumnName;
-         myRowTimeToLive     = rowTimeToLive;
-         myFilterFrequency   = filterFrequency;
-         myNextFilter        = DateTime.MinValue;
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+         myVsd = vsd;
+         _time_col = rowTimeColumnName;
+         _ttl = rowTimeToLive;
+         myFilterFrequency = filterFrequency;
+         myNextFilter = DateTime.MinValue;
 
          // add worker
          myVsd.addWorker(this);
@@ -41,32 +70,25 @@ namespace BCG
       public event FilterChangeDelegate onFilterChange;
 
       #region IStreamFilter
-      public bool filter(IBlueRow row)
-      {
-         DateTime now       = DateTime.Now;
-         DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
+      public bool filter(IBlueRow row) {
+         DateTime now = DateTime.Now;
 
-         if (now > timeStamp)
-         {
-            TimeSpan span = now - timeStamp;
-
-            if (span >= myRowTimeToLive)
-            {
+         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
+         if (now > row_time) {
+            TimeSpan span = now - row_time;
+            if (span >= _ttl) {
                return (false);
             }
          }
-
          return (true);
       }
       #endregion
 
       #region IWorker
-      public void doWork()
-      {
+      public void doWork() {
          DateTime now = DateTime.Now;
 
-         if (now > myNextFilter)
-         {
+         if (now > myNextFilter) {
             // fire event
             onFilterChange();
 
@@ -76,8 +98,7 @@ namespace BCG
       #endregion
 
       #region IDisposable
-      public void Dispose()
-      {
+      public void Dispose() {
          // remove worker
          myVsd.removeWorker(this);
       }
@@ -85,10 +106,20 @@ namespace BCG
 
       #region state
       private ViewStreamDatabase myVsd;
-      private string             myRowTimeColumnName;
-      private TimeSpan           myRowTimeToLive;
-      private TimeSpan           myFilterFrequency;
-      private DateTime           myNextFilter;
+      private string _time_col;
+      private TimeSpan _ttl;
+      private TimeSpan myFilterFrequency;
+      private DateTime myNextFilter;
       #endregion
    }
+   [DataStreamUserRespresentation("timefilter")]
+   public class TimeBasedFilterStream : ViewStream {
+      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
+      :
+      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
+      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
+         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
+         mySetRealSource(new FilterStream(base_stream, filter));
+      }
+   }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs

@@ -1,102 +1,102 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
 
-namespace BCG
-{
+// namespace BCG
+// {
    
-    public class AverageGroupsStream : ViewStream
-    {
-        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-        {
+//     public class AverageGroupsStream : ViewStream
+//     {
+//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+//         {
             
-            myColToAverage = colToAverage;
-            myInput = input;
-            myKeyColumn = keyColumn;
-        }
+//             myColToAverage = colToAverage;
+//             myInput = input;
+//             myKeyColumn = keyColumn;
+//         }
 
-        private ViewStream myInput;
-        private string myColToAverage;
-        private BlueTableSchema mySchema;
-        private string myKeyColumn;
+//         private ViewStream myInput;
+//         private string myColToAverage;
+//         private BlueTableSchema mySchema;
+//         private string myKeyColumn;
 
-        protected override void myOnFirstSubscriber()
-        {
-            myInput.onChange += new ChangeDelegate(myInput_onChange);
-        }
-        protected override void myOnLastSubscriber()
-        {
-            myInput.onChange -= myInput_onChange;
-        }
-        private void myInput_onChange(object newValue)
-        {
-            if (mySchema == null)
-                myBuildSchema(((DataPacket)newValue).schema);
+//         protected override void myOnFirstSubscriber()
+//         {
+//             myInput.onChange += new ChangeDelegate(myInput_onChange);
+//         }
+//         protected override void myOnLastSubscriber()
+//         {
+//             myInput.onChange -= myInput_onChange;
+//         }
+//         private void myInput_onChange(object newValue)
+//         {
+//             if (mySchema == null)
+//                 myBuildSchema(((DataPacket)newValue).schema);
 
-            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
 
-            if (myLastValue == null)
-                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-            else
-            {
-                IBlueRowCollection added, deleted;
-                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-                myFireEvent(new DataPacket(newRows, added, deleted));
-            }
-        }
+//             if (myLastValue == null)
+//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+//             else
+//             {
+//                 IBlueRowCollection added, deleted;
+//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+//                 myFireEvent(new DataPacket(newRows, added, deleted));
+//             }
+//         }
 
-        private void myBuildSchema(IBlueTableSchema schema)
-        {
-            List<ColumnInfo> columns = new List<ColumnInfo>();
+//         private void myBuildSchema(IBlueTableSchema schema)
+//         {
+//             List<ColumnInfo> columns = new List<ColumnInfo>();
 
-            foreach (ColumnInfo ci in schema.columnInfo)
-                columns.Add(ci);
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 columns.Add(ci);
 
-            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-        }
+//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+//         }
 
 
 
-        private IBlueRowCollection myEvaluate(DataPacket packet)
-        {
+//         private IBlueRowCollection myEvaluate(DataPacket packet)
+//         {
          
-            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-            IDictionary<string, int> colToCount = new Dictionary<string, int>();
-            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
+//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
 
-            foreach (IBlueRow r in packet.newRows)
-            {
-                double thisVal = Convert.ToDouble(r[myColToAverage]);
+//             foreach (IBlueRow r in packet.newRows)
+//             {
+//                 double thisVal = Convert.ToDouble(r[myColToAverage]);
 
-                string keyVal = r[myKeyColumn].ToString();
-                if (!colToCount.ContainsKey(keyVal))
-                {
-                    colToRunningTotal[keyVal] = thisVal;
-                    colToCount[keyVal] = 1;
-                }
-                else
-                {
-                    colToRunningTotal[keyVal] += thisVal;
-                    colToCount[keyVal] += 1;
-                }
-                colToLastRow[keyVal] = r;
-            }
+//                 string keyVal = r[myKeyColumn].ToString();
+//                 if (!colToCount.ContainsKey(keyVal))
+//                 {
+//                     colToRunningTotal[keyVal] = thisVal;
+//                     colToCount[keyVal] = 1;
+//                 }
+//                 else
+//                 {
+//                     colToRunningTotal[keyVal] += thisVal;
+//                     colToCount[keyVal] += 1;
+//                 }
+//                 colToLastRow[keyVal] = r;
+//             }
 
-            BlueTable result = new BlueTable(mySchema);
-            foreach (string key in colToCount.Keys)
-            {
-                object[] values = new object[mySchema.columnInfo.Length];
+//             BlueTable result = new BlueTable(mySchema);
+//             foreach (string key in colToCount.Keys)
+//             {
+//                 object[] values = new object[mySchema.columnInfo.Length];
 
-                colToLastRow[key].CopyTo(values, 0);
-                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-                result.addRow(new BlueRow(mySchema, values));
-            }
+//                 colToLastRow[key].CopyTo(values, 0);
+//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+//                 result.addRow(new BlueRow(mySchema, values));
+//             }
 
-            return result;
-        }
+//             return result;
+//         }
 
       
-    }
-}
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs

@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("Average")]
+   public class AverageNonIncGroupsStream : ViewStream {
+      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
+         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
+                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
+         mySetRealSource(rs);
+      }
+
+      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
+         IBlueRow inp_row = g_rows.ElementAt(0);
+         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
+         double sum = 0.0;
+         inp_vals.ToList().ForEach(d => sum += d);
+         string out_col_name = $"average_{inp_col_name}";
+         double average = sum / inp_vals.Count();
+         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
+         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
+         out_row.setValue(out_col_name, average);
+         List<IBlueRow> rv = new List<IBlueRow>();
+         rv.Add(out_row);
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -0,0 +1,91 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   using ZstdSharp.Unsafe;
+
+   public enum GroupByBundleOp {
+      Mean,
+      Median,
+      Mode,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum
+   }
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;
+
+      public static GroupByBundle parse(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new GroupByBundle(rv);
+      }
+
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
+         operations = operations_;
+      }
+
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            GroupByBundleOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
+         switch (op) {
+            case GroupByBundleOp.Max:
+               return col_vals.Max<double>();
+            case GroupByBundleOp.Min:
+               return col_vals.Min<double>();
+            case GroupByBundleOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case GroupByBundleOp.Sum:
+               return col_vals.Sum();
+            case GroupByBundleOp.First:
+               return col_vals.First();
+            case GroupByBundleOp.Last:
+               return col_vals.Last();
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -4,18 +4,14 @@ using BCG.Utilities;
 using System.Collections.Generic;
 using System.Linq;
 
-namespace BCG
-{
-   public class GroupByStream : MethodDrivenStream
-   {
-      private class MyFuncGroupByEvaluator : IGroupByEvaluator
-      {
+namespace BCG {
+   public class GroupByStream : MethodDrivenStream {
+      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         {
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
-            _groupby_func = groupby_func;            
+            _groupby_func = groupby_func;
          }
          public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
 
@@ -29,7 +25,7 @@ namespace BCG
       /// Serves as  a quick way to access all rows belonging to the same group
       /// </summary>
       private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable ;
+      private RowDictionaryTable myBookKeepingTable;
       private IGroupByEvaluator myGroupByEvaluator;
       private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
       //can map a schema to another
@@ -37,112 +33,98 @@ namespace BCG
       private int myIdGenerator = 0;
 
       public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
-      { }
+         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
-         : this (stream, groupByEvaluator, true)
-      {}
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
+         : this(stream, groupByEvaluator, true) { }
 
-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
-         : base (stream)
-      {
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
+         : base(stream) {
          myGroupByEvaluator = groupbyEvaluator;
          myIdGuard = idGuard;
       }
 
-      protected override void myHandleFirstCall ()
-      {
+      protected override void myHandleFirstCall() {
          mySchema = myLastArrivedPacket.schema;
-         myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
-         myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, 
+         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
+         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
             myGroupByEvaluator.groupByColumns);
-         GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
-         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
-         {
-            IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
-               rowGroup, new BlueTable (mySchema)));
+         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
+         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
+            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
+               rowGroup, new BlueTable(mySchema)));
             if (newRow != null)
-               temp.addRow (newRow, myGetGroupString (rowGroup));
+               temp.addRow(newRow, myGetGroupString(rowGroup));
          }
-         myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
+         myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
       }
 
       /// <summary>
       /// Adding need not to anything as deletion already takes care
       /// of addition
       /// </summary>
-      protected override void myHandleIncrementalAdd()   {}
-      
+      protected override void myHandleIncrementalAdd() { }
+
       /// <summary>
       /// Both deletes and adds are handled here to make the intermediate results not
       /// appear in the spitted fired DataPacket (note this is done only in GroupByStream 
       /// because of the nature of it)
       /// </summary>
-      protected override void myHandleIncrementalDelete()   
-      {
+      protected override void myHandleIncrementalDelete() {
          myFireSwitch = false;
          RowDictionaryTable delGroups, addGroups;
          //form groups for deleted and added rows
-         myInitIncrementalChangeData (out delGroups, out addGroups);
-         ArrayList handledAddGroups = new ArrayList ();
+         myInitIncrementalChangeData(out delGroups, out addGroups);
+         ArrayList handledAddGroups = new ArrayList();
          string groupRepString;
 
-         foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
             // BlueEventLog.logAssert (delGroup.Count > 0);
-            groupRepString = myGetGroupString (delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
-            handledAddGroups.Add (addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
-            IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
-            myFireSwitch |= myMergeRow (newRow, groupRepString);           
+            groupRepString = myGetGroupString(delGroup);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
+            handledAddGroups.Add(addGroup);
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
+            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            myFireSwitch |= myMergeRow(newRow, groupRepString);
          }
 
-         foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
             // BlueEventLog.logAssert (addGroup.Count > 0);
-            if (!handledAddGroups.Contains (addGroup))
-            {
+            if (!handledAddGroups.Contains(addGroup)) {
                //this is the case when there are rows added to a group and none
                //have been removed, hence it 'escapes' from being dealt with in 
                //the loop above
 
-               groupRepString = myGetGroupString (addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
+               groupRepString = myGetGroupString(addGroup);
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
                // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, 
-                  new BlueTable (aggGroup.schema)));
-               myFireSwitch |= myMergeRow (newRow, groupRepString);
+               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
+                  new BlueTable(aggGroup.schema)));
+               myFireSwitch |= myMergeRow(newRow, groupRepString);
             }
          }
       }
 
-      private bool myMergeRow (IBlueRow newRow, string groupRepString)
-      {
-         IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
+      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
+         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
          //BlueEventLog.logAssert (newRow != null || existingRow != null);
          if (existingRow == null && newRow == null)
             return false;
-         else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
+         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
             return false;
-         else if (existingRow == null)
-         {
+         else if (existingRow == null) {
             //row added for the first time
-            ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-            ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
             return true;
-         }
-         else
-         {
-            ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
-            ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
-            if (newRow != null)
-            {
-               ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-               ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+         } else {
+            ((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
+            ((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
+            if (newRow != null) {
+               ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+               ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
             }
             return true;
          }
@@ -154,77 +136,68 @@ namespace BCG
       /// </summary>
       /// <param name="delGroups"></param>
       /// <param name="addGroups"></param>
-      private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
-      {
+      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
          delGroups =
-            new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
          addGroups =
-            new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
          foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
-            myBookKeepingTable.removeRow (deletedRow);
+            myBookKeepingTable.removeRow(deletedRow);
          foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
-            myBookKeepingTable.addRow (addedRow);
-         myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
+            myBookKeepingTable.addRow(addedRow);
+         myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
       }
 
-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
          if (myBookKeepingTable.Count == 0)
             return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
-         myBookKeepingTable.clear ();
-         myGroupByEvaluator.clear ();
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
+         myBookKeepingTable.clear();
+         myGroupByEvaluator.clear();
       }
 
-      private IBlueRow myEvaluate (DataPacket dp)
-      {
-         IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
-         return evalRow == null ? null : 
+      private IBlueRow myEvaluate(DataPacket dp) {
+         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
+         return evalRow == null ? null :
             myIdGuard ?
-            BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
+            BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
       }
 
-      private string myGetGroupString (IBlueRowCollection group)
-      {
-         return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
+      private string myGetGroupString(IBlueRowCollection group) {
+         return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
       }
 
-      protected override  void myOnLastSubscriber () 
-      {
+      protected override void myOnLastSubscriber() {
          //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream 
          //gets resubscribed, we start the process of building data again, and treat the callback as the first 
          //callback.
-         myInit ();
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myInit();
+         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override void myInit ()
-      {
-         base.myInit ();
-         if (myBookKeepingTable != null)
-         {
+      protected override void myInit() {
+         base.myInit();
+         if (myBookKeepingTable != null) {
             //to avoid calling init before the first callback
-            myBookKeepingTable.clear ();
-            myGroupByEvaluator.clear ();
+            myBookKeepingTable.clear();
+            myGroupByEvaluator.clear();
          }
       }
 
-      public static string getRepString (DataPacket dp, string[] colNames)
-      {
-         return BlueDataUtility.rowToString (getRepRow (dp), colNames);
+      public static string getRepString(DataPacket dp, string[] colNames) {
+         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow (DataPacket dp)
-      {
+      public static IBlueRow getRepRow(DataPacket dp) {
          if (dp.newRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.newRows, 0);
+            return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.addedRows, 0);
+            return BlueDataUtility.extractRow(dp.addedRows, 0);
          else if (dp.deletedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.deletedRows, 0);
+            return BlueDataUtility.extractRow(dp.deletedRows, 0);
          // BlueEventLog.logAssert (false, "Illegal Data Packet");
          return null;
       }
@@ -237,22 +210,21 @@ namespace BCG
       /// <param name="newValue"></param>
       /// <param name="isAdd"></param>
       /// <returns></returns>
-      public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
-      {
-         if (lastValue == null || lastValue.Equals (DBNull.Value))
+      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
+         if (lastValue == null || lastValue.Equals(DBNull.Value))
             return newValue;
          // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
-            // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
-            // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
+         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
+         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
          Object retVal = null;
-         if (newValue.GetType().Equals (typeof(int)))
+         if (newValue.GetType().Equals(typeof(int)))
             retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
-         else if (newValue.GetType().Equals (typeof(Double)))
+         else if (newValue.GetType().Equals(typeof(Double)))
             retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
-         else if (lastValue.GetType().Equals (typeof(Decimal)))
+         else if (lastValue.GetType().Equals(typeof(Decimal)))
             retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
-         else{}
-            // BlueEventLog.logAssert (false, "Illegal type");
+         else { }
+         // BlueEventLog.logAssert (false, "Illegal type");
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -22,8 +22,8 @@ namespace BCG {
             IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
             if (maxRow != null) {
                 bts.Log.assert(currMaxRow != null);
-                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
-                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
                     maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
                     if (maxRow == null)
                         myOutputTable.removeRow(hashString);
@@ -34,7 +34,7 @@ namespace BCG {
             maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
             // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
             // otherwise if maxRow > currMaxRow then its replaced
-            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
                 myOutputTable.addRow(maxRow, hashString);
             return myProject(myOutputTable.getRow(hashString));
         }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -10,6 +10,6 @@ namespace BCG {
         }
 
         public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
     }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -18,7 +18,7 @@ namespace BCG {
             if (minRow != null) {
                 // BlueEventLog.logAssert (currMinRow != null);
                 // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
+                if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
                     minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
                     if (minRow == null)
                         myOutputTable.removeRow(stringRep);
@@ -27,7 +27,7 @@ namespace BCG {
                 }
             }
             minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
-            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
                 myOutputTable.addRow(minRow, stringRep);
             return myOutputTable.getRow(stringRep);
         }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,53 +1,108 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using mbts;
-using mbts.spark;
+using System.Text.Json;
 
-namespace BCG
-{
-   public class NonIncGroupByStream : ViewStream
-   {
+using bts;
+using bts.SilverPine;
+
+namespace BCG {
+   [DataStreamUserRespresentation("aggregate")]
+   public class NonIncGroupByStream : ViewStream {
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
       private ViewStream _main_stream;
       private INonIncGroupByEvaluator _groupby_func;
       private BlueRowHasher _row_hasher;
       private string[] _groupby_cols;
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();
+      private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
-      {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
          _groupby_func = groupby_func;
          _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
       protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
 
       protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
 
-      private void _main_stream_onchange(object nv)
-      {
+      private void _main_stream_onchange(object nv) {
          DataPacket dp = nv as DataPacket;
          ///grouping together
-         _timer.start();
-         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.stop();
-         // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         _timer.Restart();
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         _timer.Stop();
+         var groupElapsedMs = _timer.ElapsedMilliseconds;
+         if (groupElapsedMs >= 50) {
+            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
+         }
+         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
-         _timer.start();
+         _timer.Restart();
          IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.stop();
-         _n_total_fires++;         
-         if (_timer.lastIntervalTime > 50)
-         {
+         _timer.Stop();
+         var applyElapsedMs = _timer.ElapsedMilliseconds;
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
             _n_expensive_fires++;
-            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
-         }         
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
-      }      
+            Log.DataFlow.info(
+               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+      }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs

@@ -27,7 +27,8 @@ namespace BCG
             string prependString = isLeft ? "left" : "right";
             return String.Concat (prependString, colName).GetHashCode ();
          }
-         public override bool Equals (object obj)  {return GetHashCode () == obj.GetHashCode ();}
+         public override bool Equals (object? obj)  =>
+            GetHashCode() == obj?.GetHashCode();
       }
 
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs

@@ -10,9 +10,9 @@ namespace BCG
 {
    public class JoinStream : ViewStream
    {
-      protected InternalJoinStream myIjs;
-      protected ProjectStream myPs;
-      protected ColumnMap myLeftCm, myRightCm;
+      protected InternalJoinStream? myIjs;
+      protected ProjectStream? myPs;
+      protected ColumnMap? myLeftCm, myRightCm;
       protected bool myIsProject = true;
       private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
       

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,15 +1,16 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using mbts;
-using mbts.spark;
+using bts;
+using bts.collections;
 
-namespace BCG
-{
-   public class NonIncJoinBookkeeper : IDisposable
-   {
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;
+
+namespace BCG {
+   public class NonIncJoinBookkeeper : IDisposable {
       public Action on_new_data;
       private readonly NonIncJoinSpec _join_spec;
       private readonly int _join_position;
@@ -26,54 +27,82 @@ namespace BCG
 
       private int[] _join_col_indices = null,
                      _data_col_indices = null;
-      
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
-      {
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
          _join_spec = the_join_spec;
          _join_position = join_position;
          _start_index_in_output = start_index_in_output;
          _row_hasher = row_hasher;
          _is_left_most = _join_position == 0;
          _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
          _join_spec.stream.onChange += _on_stream_cb;
       }
 
-      public void _on_stream_cb(object dp_obj)
-      {
-         HighPerformanceTimer timer = new HighPerformanceTimer();
-         timer.start();
+      public void _on_stream_cb(object dp_obj) {
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
-         if (_schema == null)
-         {
+         _fill_missing_data(dp);
+         if (_schema == null) {
             _schema = dp.schema;
             _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
             _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
          }
 
-         if (_is_left_most)
-         {
+         if (_is_left_most) {
             _row_list.Clear();
             foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
-         }
-         else
-         {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
             if (_is_tag_along)
                _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else
-            {
+            else {
                _row_map.Clear();
                foreach (IBlueRow r in dp.newRows)
                   _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
             }
          }
-         timer.stop();
+         timer.Stop();
          _n_total_fires++;
-         if (timer.lastIntervalTime > 50)
-         {
+         if (timer.ElapsedMilliseconds >= 50) {
             _n_expensive_fires++;
-            string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
-            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
+            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
          }
          if (on_new_data != null)
             on_new_data();
@@ -81,31 +110,25 @@ namespace BCG
 
       public bool is_ready { get { return _schema != null; } }
 
-      public void constribute_cols_to_schema(List<ColumnInfo> cis)
-      {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
          for (int i = 0; i < _join_spec.data_col_names.Length; i++)
             cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
       }
 
       public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
 
-      public void copy_master_row_to_output_row(int row_index, object[] output_row)
-      {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
          IBlueRow this_row = _row_list[row_index].second;
          BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
       }
 
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
-      {
-         if (_is_tag_along)
-         {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
             if (_solitary_row != null)
                BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
             else
                copy_dbnull_to_output_row(output_row);
-         }
-         else
-         {
+         } else {
             IBlueRow this_row = null;
             if (_row_map.TryGetValue(row_hash, out this_row))
                BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -114,8 +137,7 @@ namespace BCG
          }
       }
 
-      private void copy_dbnull_to_output_row(object[] output_row)
-      {
+      private void copy_dbnull_to_output_row(object[] output_row) {
          for (int i = 0; i < _data_col_indices.Length; i++)
             output_row[i + _start_index_in_output] = DBNull.Value;
       }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -3,63 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+
 using mbts;
 
-namespace BCG
-{
-   public class NonIncJoinSpec
-   {
+namespace BCG {
+   // this a collection of data, theres no joining logic in here
+   public class NonIncJoinSpec {
       public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
       public string[] join_col_names, data_col_names, projected_col_names;
 
       public NonIncJoinSpec() { }
 
       public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
-      { }
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
 
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
-      {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
          stream = the_stream;
          join_col_names = the_join_col_names;
          data_col_names = the_data_col_names;
          projected_col_names = the_projected_col_names;
       }
 
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
-      {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
          List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs)
-         {
-            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
             string[] visible_cols = r.third.Split(' ');
             List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols)
-            {
+            foreach (string s in visible_cols) {
                string[] toks = s.Split(':');
-               if (toks.Length == 1)
-               {
+               if (toks.Length == 1) {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[0]);
-               }
-               else if (toks.Length == 2)
-               {
+               } else if (toks.Length == 2) {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[1]);
                }
             }
-            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
          }
-         return rv.ToArray ();
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
       }
    }
-
-   //[DataStreamUserRespresentation("ninctest")]
-   //public class NonIncJoinTestStream : ViewStream
-   //{
-   //   public NonIncJoinTestStream (string key_cols, string data_cols)
-   //   {
-   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
-   //   }
-   //}
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -1,16 +1,14 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using mbts;
-using mbts.spark;
+using bts;
 
-namespace BCG
-{
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace BCG {
    [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
-   {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
       private readonly NonIncJoinSpec[] _join_specs;
       private readonly INonIncJoinInlineMapper _output_mapper;
       private readonly BlueRowHasher _row_hasher;
@@ -28,24 +26,19 @@ namespace BCG
                {
                   new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                   new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               })
-      { }
+               }) { }
 
       public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
-      { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
 
       public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                               params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
-      { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
 
       public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
-      { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
 
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-      {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
          _join_specs = join_specs;
          _output_mapper = output_mapper;
          _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -53,12 +46,11 @@ namespace BCG
          _vsd = vsd;
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++)
-         {
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -66,10 +58,11 @@ namespace BCG
          if (_output_mapper != null)
             _output_mapper.onMapperChange += _on_mapper_change;
          _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             d.Dispose();
          if (_output_mapper != null)
@@ -77,30 +70,26 @@ namespace BCG
          _vsd.removeWorker(this);
       }
 
-      private void _stream_callback()
-      {
+      private void _stream_callback() {
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
          _is_dirty = true;
          doWork();
       }
 
-      private void _on_mapper_change()
-      {
+      private void _on_mapper_change() {
          _is_dirty = true;
          doWork();
       }
 
-      private bool _is_ready()
-      {
+      private bool _is_ready() {
          bool rv = true;
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             rv &= d.is_ready;
          return rv;
       }
 
-      private void _construct_schema()
-      {
-         if (_output_schema == null)
-         {
+      private void _construct_schema() {
+         if (_output_schema == null) {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
             foreach (NonIncJoinBookkeeper d in _book_keeping_data)
                d.constribute_cols_to_schema(cis);
@@ -109,28 +98,23 @@ namespace BCG
       }
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire()
-      {
-         if (_is_ready())
-         {
+      private void _form_and_fire() {
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
             _construct_schema();
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
+            var timer = Stopwatch.StartNew();
             IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
             List<object[]> output_rows = new List<object[]>();
 
-            for (int i = 0; i < left_row_list.Count; i++)
-            {
+            for (int i = 0; i < left_row_list.Count; i++) {
                object[] or = new object[_output_schema.columnInfo.Length];
                or[0] = i;
                output_rows.Add(or);
             }
 
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
-            {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
                NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++)
-               {
+               for (int i = 0; i < left_row_list.Count; i++) {
                   if (ji == 0)
                      bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                   else
@@ -138,44 +122,38 @@ namespace BCG
                }
             }
             IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.stop();
+            timer.Stop();
             _n_total_fires++;
-            if (timer.lastIntervalTime >= 250)
-            {
-               _n_expensive_fires++;
-               string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
-               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
-            }
-            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
       }
 
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
-      {
-         if (_output_mapper != null)
-         {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
             IBlueRowCollection output_rows;
-            timer.start();
+            var timer = Stopwatch.StartNew();
             output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.stop();
-            if (timer.lastIntervalTime >= 50)
-            {
-               // string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
-               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
+            timer.Stop();
+            if (timer.ElapsedMilliseconds >= 50) {
+               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
             }
             return output_rows;
-         }
-         else
-         {
+         } else {
             return unmapped_rows;
          }
       }
 
-      public void doWork()
-      {
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
-         {
+      public void doWork() {
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;
             _form_and_fire();

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs

@@ -0,0 +1,68 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+   using System.Diagnostics;
+
+   using ZstdSharp.Unsafe;
+
+   // functon that joins the 2 rows and returns the outputput row. The function
+   // can return a null which means the 2 rows dont need to be joined
+   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
+   public class ProximityJoinStream : ViewStream, IWorker {
+      private ViewStream _lvs, _rvs;
+      private JoinFunc _join_func;
+      private bool _dirty = false;
+      private Stopwatch _stopwatch = new Stopwatch();
+      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
+      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
+         _lvs = lvs;
+         _rvs = rvs;
+         _join_func = join_func;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _lvs.onChange += _handle_data;
+         _rvs.onChange += _handle_data;
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _lvs.onChange -= _handle_data;
+         _rvs.onChange -= _handle_data;
+         _vsd.removeWorker(this);
+      }
+
+      private void _handle_data(object dp_obj) {
+         _dirty = true;
+         doWork();
+      }
+
+      public void doWork() {
+         Func<bool> all_data_revcd = () => {
+            return _lvs.rows != null && _rvs.rows != null;
+         };
+         _stopwatch.Start();
+         if (_dirty && all_data_revcd()) {
+            _dirty = false;
+            BlueTable rv = null;
+            foreach (IBlueRow lr in _lvs.rows) {
+               foreach (IBlueRow rr in _rvs.rows) {
+                  IBlueRow resr = _join_func(lr, rr);
+                  if (resr == null)
+                     continue;
+                  rv = rv ?? new BlueTable(resr.schema);
+                  rv.addRow(resr);
+               }
+            }
+            _stopwatch.Stop();
+            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
+            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs

@@ -0,0 +1,47 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.CodeDom;
+   using System.Configuration;
+   using System.Reactive.Disposables;
+
+   public class TimeProximityJoinStream : ViewStream {
+      private int _id = 0;
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       string pre_join_period, string post_join_period) :
+            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
+            TimeBasedFilter.parse_duration(post_join_period)) { }
+
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
+         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
+            var id = 0;
+            lock (this) {
+               id = _id++;
+            }
+            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
+            DateTime ldt = lr.get_datetime(l_time_col);
+            DateTime rdt = rr.get_datetime(r_time_col);
+            TimeSpan diff = ldt - rdt;
+            if (ldt < rdt) {
+               if (diff.Duration() > pre_join_period) {
+                  return null;
+               }
+            } else {
+               if (diff > post_join_period)
+                  return null;
+            }
+            return create_row(lr, rr);
+         };
+
+         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
+         mySetRealSource(rstr);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs

@@ -1,11 +1,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using BCG.Utilities;
 using mbts;
-using mbts.spark;
 using Spark.Core.Collections;
 
 namespace BCG
@@ -226,7 +222,8 @@ namespace BCG
 
       #region IDictionary Members
 
-      public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
+      public void Add(object key, object? value) =>
+         throw new NotSupportedException("IBlueRow is readonly");
 
       public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
 
@@ -256,10 +253,9 @@ namespace BCG
          }
       }
 
-      public object this[object key]
-      {
-         get { return key is string ? this[key.ToString()] : this[(int)key]; }
-         set { throw new NotSupportedException("IBlueRow is readonly"); }
+      public object? this[object key] {
+         get => key is string str ? this[str] : this[(int)key];
+         set => throw new NotSupportedException("IBlueRow is readonly");
       }
 
       #endregion
@@ -294,7 +290,6 @@ namespace BCG
       protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
                                           myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
       private bool myManageUnmappedLeftRows = false;
-      protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
       protected string myLeftProj = String.Empty, myRightProj = String.Empty;
       protected IBlueTableSchema myLeftSchema, myRightSchema;
       protected JoinProjectHelper myJoinProjectHelper;
@@ -361,7 +356,6 @@ namespace BCG
 
       private void myHandleStreamChanges(DataPacket dp, bool isLeft)
       {
-         myTimer.start();
          myInitSchema(dp.schema, isLeft);
          BlueTable dels = null, adds = null;
          if (myState != null)
@@ -371,7 +365,6 @@ namespace BCG
          }
          myDeleteRows(dp.deletedRows, isLeft, myState, dels);
          myAddRows(dp.addedRows, isLeft, myState, adds);
-         myTimer.stop();
          if (myState != null)
          {
             myFireEvent(new DataPacket(myState, adds, dels));

src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs

@@ -91,7 +91,8 @@ namespace BCG
          }
       }
 
-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add (object key, object? value) =>
+         throw new NotSupportedException ("The IBlueRow is readonly");
 
       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
 

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -1,51 +1,96 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;
+
+using Amazon.Runtime.Internal.Util;
+
 using BCG;
 using BCG.Utilities;
 
-namespace BCG
-{
-   public class LiteralStream : ViewStream
-   {
-      public LiteralStream(string tableName, string schema, string value)
-      {
-         string [] columns = schema.Split(' ');
+using bts;
+
+namespace BCG {
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');
 
 
-         ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id",0,typeof(int));
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
          int n = 1;
-         foreach (string c in columns)
-         {
-            string [] columnAndType = c.Split('/');
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
             if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c,n,typeof(double));
+               ci[n] = new ColumnInfo(c, n, typeof(double));
             else
-               ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
             n++;
          }
 
 
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
-         string [] rows = value.Split(';');
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
          n = 0;
-         foreach (string r in rows) 
-            try 
-            {
-               if(r == string.Empty)
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
                   continue;
                t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));   
-            }
-            catch (Exception e)
-            {
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
                throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
             }
 
          mySetRealSource(new BlueRowCollectionStaticStream(t));
       }
 
-      public LiteralStream(string schema, string value) : this ("literal",schema, value)
-      {}
+      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+
+      // public LiteralStream(string value) {
+
+      // }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs

@@ -1,24 +1,24 @@
-using System;
+// using System;
 
-namespace BCG
-{
-   public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
-   {
-      IBlueTableSchema mySchema;
+// namespace BCG
+// {
+//    public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
+//    {
+//       IBlueTableSchema mySchema;
 
-      public IBlueRow map (IBlueRow row)
-      {
-         if (mySchema == null)
-         {
-            ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
-            int i = 0;
-            foreach (ColumnInfo ci in row.schema.columnInfo)
-               cis[i++] = ci.colType == null ?
-                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
-                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
-            mySchema = new BlueTableSchema (row.schema.tableName, cis);
-         }
-         return new BlueRow (mySchema, row);
-      }
-   }
-}
+//       public IBlueRow map (IBlueRow row)
+//       {
+//          if (mySchema == null)
+//          {
+//             ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
+//             int i = 0;
+//             foreach (ColumnInfo ci in row.schema.columnInfo)
+//                cis[i++] = ci.colType == null ?
+//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
+//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
+//             mySchema = new BlueTableSchema (row.schema.tableName, cis);
+//          }
+//          return new BlueRow (mySchema, row);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -0,0 +1,72 @@
+using BCG;
+
+using MongoDB.Driver.Core.WireProtocol.Messages;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
+// the func returns the mapped row (ie mapped version of the index row)
+using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
+
+namespace bts {
+   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
+   // can use the state fo all rows.
+   public class GlobalMapStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _sort_col_name, _group_by_col;
+      private GlobalMap _map_func;
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
+         _base_stream = base_stream;
+         _sort_col_name = sort_col_name;
+         _group_by_col = group_by_col;
+         _map_func = map_func;
+      }
+
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
+      this(base_stream, sort_col_name, null, map_func) {
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_stream_update;
+      }
+
+      void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
+         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
+         :
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
+         BlueTable out_table = null;
+         foreach (var kv in groups) {
+            List<IBlueRow> group_rows = kv.Value;
+            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
+            uint i = 0;
+            foreach (IBlueRow r in group_rows) {
+               IBlueRow nr = _map_func(group_rows, i++);
+               out_table = out_table ?? new BlueTable(nr.schema);
+               out_table.addRow(nr);
+            }
+         }
+
+
+         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
+         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
+         // uint i = 0;
+
+         // BlueTable out_table = null;
+         // foreach (IBlueRow r in rows) {
+         //    IBlueRow nr = _map_func(rows, i++);
+         //    out_table = out_table ?? new BlueTable(nr.schema);
+         //    out_table.addRow(nr);
+         // }
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -1,15 +1,10 @@
-using System;
-using System.Data;
-using System.Collections;
-
 using Bts.DataStream;
 
-using mbts;
-using mbts.spark;
 using mbts.config;
 
-namespace BCG
-{
+using System;
+
+namespace BCG {
 
 
    /// <summary>
@@ -21,24 +16,45 @@ namespace BCG
    [DataStreamUserRespresentation (false)]
    public class MapStream : MethodDrivenStream
    {
-      private HighPerformanceTimer t = new HighPerformanceTimer ();
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
 
-      public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
-      {
-         if (myUseNewMap)
-            mySetRealSource (new SuperMapStream (vs, mapper));
-         else
-            myMapper = mapper;
+      public class FuncWrapper : IStreamMapper {
+         private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func;
+         Func<IBlueRow, IBlueRow> _row_map_func;
+
+         public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
+         Func<IBlueRow, IBlueRow> row_map_func) {
+            _schema_map_func = schema_map_func;
+            _row_map_func = row_map_func;
+         }
+         public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+            return _schema_map_func(input_schema);
+         }
+         public IBlueRow map(IBlueRow row) {
+            return _row_map_func(row);
+         }
+
       }
-      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
-      {
+
+      public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
          if (myUseNewMap)
             mySetRealSource(new SuperMapStream(vs, mapper));
          else
-         {
+            myMapper = mapper;
+      }
+
+      public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
+                                       Func<IBlueRow, IBlueRow> row_map_func)
+                                       :
+                                       this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
+      }
+
+      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
+         if (myUseNewMap)
+            mySetRealSource(new SuperMapStream(vs, mapper));
+         else {
             mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
             myChangeableMapper = mapper;
          }
@@ -47,77 +63,68 @@ namespace BCG
       /// <summary>
       /// Gets or sets the value of the mapper. Setting the value will result in re
       /// </summary>
-      private void myMapperChanged () 
-      {
+      private void myMapperChanged() {
          if (myLastArrivedPacket != null)
-            myChangeHandler (new DataPacket (myLastArrivedPacket.newRows, 
+            myChangeHandler(new DataPacket(myLastArrivedPacket.newRows,
                                              myLastArrivedPacket.newRows,
                                              myLastArrivedPacket.newRows));
       }
 
 
-      protected override void myHandleFirstCall ()
-      {
-         
-         IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
-         myLastFiredPacket = new DataPacket ();
-         myLastFiredPacket.newRows = new BlueTable (outputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
+      protected override void myHandleFirstCall() {
+
+         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
+         myLastFiredPacket = new DataPacket();
+         myLastFiredPacket.newRows = new BlueTable(outputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(outputSchema);
          foreach (IBlueRow row in myLastArrivedPacket.newRows)
-            ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
+            ((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row));
          myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
       }
 
-      protected override void myHandleIncrementalAdd()
-      {
+      protected override void myHandleIncrementalAdd() {
          //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows)
-         {
-            IBlueRow newRow = myMapper.map (row);
-            (myLastFiredPacket.newRows as BlueTable).addRow (newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
+            IBlueRow newRow = myMapper.map(row);
+            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
+            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
          }
          //t.stop ();
          //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
          //System.Diagnostics.Trace.WriteLine (" ");
       }
 
-      protected override void myHandleIncrementalDelete()
-      {  
+      protected override void myHandleIncrementalDelete() {
          //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
-         {
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
             IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow (row);
+            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
+            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
          }
          //t.stop ();
          //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
       }
-     
 
-      protected override void myOnFirstSubscriber() 
-      {
-         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
+
+      protected override void myOnFirstSubscriber() {
+         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override  void myOnLastSubscriber () 
-      {
-         myInit ();
+      protected override void myOnLastSubscriber() {
+         myInit();
          if (myStream != null)
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-         myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
       }
 
-      
+
    }
 
    /*

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -1,12 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+
 using BCG.Utilities;
 using BCG.Utilities.General;
+
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// SuperMap gives the user the ability
    /// to map one row to one or more rows
@@ -17,13 +18,11 @@ namespace BCG
    /// can chage itself and raise an event causing
    /// a remapping to happen
    /// </summary>
-   public class SuperMapStream : ViewStream
-   {
+   public class SuperMapStream : ViewStream {
       #region super mapper that wraps around stream mapper
-      private class MyMappingWrapper : ISuperStreamMapper
-      {
+      private class MyMappingWrapper : ISuperStreamMapper {
          private IStreamMapper myMapper;
-         
+
          public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
 
          #region ISuperStreamMapper Members
@@ -32,8 +31,7 @@ namespace BCG
 
          public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
 
-         public event MapperChangeDelegate onMapperChange
-         {
+         public event MapperChangeDelegate onMapperChange {
             add { }
             remove { }
          }
@@ -42,21 +40,18 @@ namespace BCG
       }
       #endregion
 
-      private class MyFuncWrapper : ISuperStreamMapper
-      {
+      private class MyFuncWrapper : ISuperStreamMapper {
          private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
          private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
          public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
-                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         {
+                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) {
             _get_schema = get_schema;
             _map_rows = map_rows;
          }
 
          public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
 
-         public IBlueRow[] map(IBlueRow r)
-         {
+         public IBlueRow[] map(IBlueRow r) {
             IEnumerable<IBlueRow> rows = _map_rows(r);
             return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
          }
@@ -76,19 +71,15 @@ namespace BCG
 
       public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
                                             Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, new MyFuncWrapper(get_schema, map_rows))
-      { }
+         : this(vs, new MyFuncWrapper(get_schema, map_rows)) { }
 
       public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, s => out_schema, map_rows)
-      { }
+         : this(vs, s => out_schema, map_rows) { }
 
       public SuperMapStream(ViewStream vs, IStreamMapper mapper)
-         : this(vs, new MyMappingWrapper(mapper))
-      { }
+         : this(vs, new MyMappingWrapper(mapper)) { }
 
-      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
-      {
+      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) {
          myStream = vs;
          myMapper = mapper;
          myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
@@ -96,8 +87,7 @@ namespace BCG
 
       protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          myStream.onChange -= new ChangeDelegate(myHandleData);
          myRows = null;
          myBaseRows = null;
@@ -105,8 +95,7 @@ namespace BCG
          //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
       }
 
-      private void myHandleData(object newValue)
-      {
+      private void myHandleData(object newValue) {
          DataPacket idp = newValue as DataPacket;
          //HighPerformanceTimer timer = new HighPerformanceTimer();
          //timer.start();
@@ -117,24 +106,19 @@ namespace BCG
          BlueTable dels = new BlueTable(myRows.schema),
                      adds = new BlueTable(myRows.schema);
 
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
             List<IBlueRow> mappedRows = myMap[(int)r["id"]];
-            foreach (IBlueRow mr in mappedRows)
-            {
+            foreach (IBlueRow mr in mappedRows) {
                dels.addRow(mr);
                myRows.deleteRow(mr);
             }
             myMap.Remove((int)r["id"]);
          }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             IBlueRow[] mappedRows = myMapper.map(r);
-            foreach (IBlueRow mr in mappedRows)
-            {
+            foreach (IBlueRow mr in mappedRows) {
                IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"]))
-               {
+               if (!myMap.ContainsKey((int)r["id"])) {
                   myMap.Add((int)r["id"], new List<IBlueRow>());
                }
                myMap[(int)r["id"]].Add(idChangedMapRow);
@@ -145,13 +129,12 @@ namespace BCG
          //timer.stop();
          // if (timer.lastIntervalTime > 20)
          //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
-         // myFireEvent(new DataPacket(myRows, adds, dels));
+         myFireEvent(new DataPacket(myRows, adds, dels));
       }
 
-      private void myHandleMapChange()
-      {
+      private void myHandleMapChange() {
          if (myBaseRows != null)
-            myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
+            myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs

@@ -0,0 +1,142 @@
+using BCG;
+using System;
+using System.IO;
+using MongoDB.Bson;
+using BCG.Utilities.General;
+using System.Linq;
+using System.Collections.Generic;
+using System.Runtime.Intrinsics.X86;
+
+
+namespace bts {
+   // listens to the underlying stream, and writes each seriialized packet in a 
+   // seperate file
+   [DataStreamUserRespresentation("dp_recorder")]
+   public class DataPacketRecorderStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _folder;
+      StreamIOSerializationController _serializer = new StreamIOSerializationController();
+      private UInt32 _n_updates = 0;
+      public DataPacketRecorderStream(ViewStream base_stream, string folder) {
+         _base_stream = base_stream;
+         _folder = folder;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_update;
+      }
+
+      private void _handle_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         byte[] ser_data = null;
+         if (_n_updates == 0) {
+            StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
+            ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
+         } else {
+            StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
+            ser_data = _serializer.serialize(new StreamIOMessage(mra));
+         }
+         Log.assert(ser_data != null);
+         _n_updates++;
+         string file_name = $"{_folder}\\{_n_updates}.bin";
+         _write_to_file(ser_data, file_name);
+         myFireEvent(dp);
+      }
+
+      private static void _write_to_file(byte[] data, string file_name) {
+         using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
+            // Write the byte array to the file
+            fs.Write(data, 0, data.Length);
+         }
+      }
+   }
+   [DataStreamUserRespresentation("dp_replay")]
+   public class DataPacketReplayStream : ViewStream, IWorker {
+      private ViewStreamDatabase _vsd;
+      private readonly string _folder_path;
+      private readonly TimeSpan _fire_period;
+      IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
+      private int _file_ptr = 0;
+      private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
+      private BlueTable _state = null;
+
+      public DataPacketReplayStream(string folder_path, string fire_period)
+      :
+      this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
+      }
+
+      public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
+         _vsd = vsd;
+         _folder_path = folder_path;
+         _fire_period = fire_period;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         MasterWorker.getInstance(false).addWorker(this, _fire_period);
+         _load_all_files();
+      }
+
+      protected override void myOnLastSubscriber() {
+         MasterWorker.getInstance(false).removeWorker(this);
+      }
+
+      public void doWork() {
+         if (_file_ptr >= _file_data.Count())
+            // we can set _file_ptr to 0 and keep going as well
+            return;
+         var curr_d = _file_data[_file_ptr++];
+         var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
+         StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
+         Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
+         switch (msg.op) {
+            case StreamIOMessage.StreamOp.CREATE: {
+                  _state = new BlueTable(msg.create_message.rows);
+                  break;
+               }
+            case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
+                  var mra = msg.multiple_remove_add_message;
+                  foreach (IBlueRow r in mra.removed_rows)
+                     _state.deleteRow(r);
+                  foreach (IBlueRow r in mra.added_rows)
+                     _state.addRow(r);
+                  break;
+               }
+         }
+         var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
+         myFireEvent(dp);
+      }
+
+      private void _load_all_files() {
+         Func<string, string> extract_file_name = (string full_path) => {
+            string fn = full_path.Split("\\").Last();
+            fn = fn.Split(".").First();
+            return fn;
+
+         };
+         string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
+
+         // Iterate over the .bin files and perform an action
+         foreach (string file in bin_files) {
+            byte[] file_data = File.ReadAllBytes(file);
+            _file_data.Add(Tuple.Create(file, file_data));
+         }
+         _file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
+            string fn1 = extract_file_name(t1.Item1);
+            string fn2 = extract_file_name(t2.Item1);
+            int fn_i1 = Int32.Parse(fn1);
+            int fn_i2 = Int32.Parse(fn2);
+            if (fn_i1 < fn_i2)
+               return -1;
+            else if (fn_i1 == fn_i2)
+               return 0;
+            else if (fn_i1 > fn_i2)
+               return 1;
+            return 0;
+         });
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -0,0 +1,227 @@
+using BCG;
+using BCG.Utilities.General;
+using System.Collections.Generic;
+
+//using MongoDB.Driver;
+
+using System;
+using System.IO;
+using System.Linq;
+
+namespace bts
+{
+    public class PubSubTestingStream : ViewStream, IWorker
+    {
+        public class PayloadMessage
+        {
+            public int msg_id = 0;
+            public string guid;
+            public Int64 travel_epoch_ms = 0;
+            public byte[] payload = null;
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
+            {
+                msg_id = msg_id_args;
+                travel_epoch_ms = send_epoch_args;
+                payload = new byte[payload_size];
+                for (int i = 0; i < payload_size; i++)
+                {
+                    payload[i] = payload_byte;
+                }
+            }
+            private PayloadMessage() { }
+            public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
+            { }
+
+            public byte[] serialize()
+            {
+                var str = new MemoryStream();
+                BinaryWriter bw = new BinaryWriter(str);
+                bw.Write(msg_id);
+                bw.Write(guid);
+                bw.Write(travel_epoch_ms);
+                bw.Write(payload.Length);
+                bw.Write(payload);
+                str.Position = 0;
+                var bytes = str.ToArray();
+                return bytes;
+            }
+
+            public static PayloadMessage deserialize(byte[] ser_bytes)
+            {
+                var str = new MemoryStream(ser_bytes);
+                BinaryReader br = new BinaryReader(str);
+                int msg_id = br.ReadInt32();
+                string guid = br.ReadString();
+                Int64 epoch = br.ReadInt64();
+                int len = br.ReadInt32();
+                var payload = br.ReadBytes(len);
+                return new PayloadMessage()
+                {
+                    msg_id = msg_id,
+                    guid = guid,
+                    travel_epoch_ms = epoch,
+                    payload = payload
+                };
+            }
+
+        }
+        //public void
+        private readonly int _msg_size;
+        private readonly TimeSpan _publish_period;
+        private DateTime _last_publish_time = DateTime.MinValue;
+        private PublisherCore _pub_core = new PublisherCore();
+        private BlueTable _table;
+        private int _n_pubs = 0;
+        private readonly int _max_pubs;
+        private int _n_updates = 0;
+        private bool _continue_sub_thread = true;
+        private ViewStreamDatabase _vsd;
+        private bool _fire_dirty = false;
+        private PayloadMessage _last_sent_msg, _last_recvd_msg;
+        private readonly int _payload_sz;
+        private const byte _payload_byte = (byte)'A';
+        private readonly string _tag_name;
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
+        {
+            _vsd = vsd;
+            _payload_sz = payload_size;
+            var schema = BlueDataUtility.form_schema("PubSubTest",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("n_pubs", TypeCode.Int32),
+            Tuple.Create("n_recvs", TypeCode.Int32),
+            Tuple.Create("last_sent_id", TypeCode.Int32),
+            Tuple.Create("last_recv_id", TypeCode.Int32),
+            Tuple.Create("last_sent_guid", TypeCode.String),
+            Tuple.Create("last_recv_guid", TypeCode.String),
+            Tuple.Create("match", TypeCode.Boolean),
+            Tuple.Create("echo_time", TypeCode.Int32));
+            _table = new BlueTable(schema);
+            _publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
+            _msg_size = payload_size;
+            _max_pubs = max_pubs;
+            _tag_name = tag_name;
+        }
+
+        public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
+        : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
+
+        protected override void myOnFirstSubscriber()
+        {
+            _vsd.addWorker(this);
+            System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
+            th.IsBackground = true;
+            th.Start();
+        }
+
+        protected override void myOnLastSubscriber()
+        {
+            _vsd.removeWorker(this);
+            _continue_sub_thread = false;
+        }
+
+        void _log_updates(IEnumerable<PubSubServerMessage> updates)
+        {
+            var idx = 0;
+            foreach (var msg in updates)
+            {
+                bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
+                idx++;
+            }
+        }
+
+        private void _subscribe_thread_func()
+        {
+            // subscription is done in another thread
+            SubscriberCore sub_core = new SubscriberCore();
+            sub_core.subscribe(_tag_name);
+            while (_continue_sub_thread)
+            {
+                // TODO recv_id
+                var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
+                if (updates == null || updates.Count() == 0)
+                    continue;
+                _log_updates(updates);
+                _n_updates += updates.Count();
+                if (updates.Count() > 0)
+                {
+                    var server_msg = updates.Last();
+                    if (server_msg.data != null && server_msg.data.Length > 0)
+                    {
+                        _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
+                        _last_recvd_msg.travel_epoch_ms = epoch_now();
+                    }
+                    else
+                    {
+                        // why are we getting empty or null data ? (server is sending data regardless?)
+                    }
+                    _fire_dirty = true;
+                }
+            }
+        }
+
+        private void _publish_next_packet()
+        {
+            if (_last_sent_msg == null)
+            {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
+                {
+                    guid = Guid.NewGuid().ToString()
+                };
+            }
+            else
+            {
+                _last_sent_msg.msg_id++;
+                _last_sent_msg.guid = Guid.NewGuid().ToString();
+                _last_sent_msg.travel_epoch_ms = epoch_now();
+            }
+            var ser_bytes = _last_sent_msg.serialize();
+            _n_pubs++;
+            _pub_core.publish("Blob", _tag_name, ser_bytes);
+            _fire_dirty = true;
+        }
+
+        private void _fire_dp()
+        {
+            object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
+            object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
+            object echo_time = DBNull.Value;
+            object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
+            object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
+            if (_last_sent_msg != null && _last_recvd_msg != null)
+            {
+                echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
+            }
+            var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
+            IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
+                                        last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
+            BlueTable table = new BlueTable(_table.schema);
+            table.addRow(nr);
+            var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        public void doWork()
+        {
+            if (DateTime.Now - _last_publish_time > _publish_period)
+                if (_n_pubs <= _max_pubs)
+                {
+                    _publish_next_packet();
+                    _last_publish_time = DateTime.Now;
+                }
+            // publish 
+            // TODO: send id
+            if (_fire_dirty)
+            {
+                _fire_dirty = false;
+                _fire_dp();
+            }
+        }
+
+        public static long epoch_now()
+        {
+            return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -0,0 +1,223 @@
+using BCG;
+using BCG.Utilities.General;
+using System.Collections.Generic;
+
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using bts.api;
+
+namespace bts
+{
+   [DataStreamUserRespresentation("pss_echo")]
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
+         public readonly string msg_id;
+         public readonly Int64 timestamp = 0;
+         public readonly byte[] payload = null;
+
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
+            var rv = new byte[payload_size];
+            for (int i = 0; i < payload_size; i++)
+            {
+               rv[i] = payload_byte;
+            }
+            return rv;
+         }
+
+         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
+         // {
+         //    guid = guid_;
+         //    timestamp = timestamp_;
+         //    payload = new byte[payload_size];
+         //    for (int i = 0; i < payload_size; i++)
+         //    {
+         //       payload[i] = payload_byte;
+         //    }
+         // }
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
+            msg_id = guid_;
+            timestamp = timestamp_;
+            payload = payload_;
+         }
+
+         private PayloadMessage() { }
+         public byte[] serialize()
+         {
+            var str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            bw.Write(msg_id);
+            bw.Write(timestamp);
+            bw.Write(payload.Length);
+            bw.Write(payload);
+            str.Position = 0;
+            var bytes = str.ToArray();
+            return bytes;
+         }
+
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
+            var str = new MemoryStream(ser_bytes);
+            BinaryReader br = new BinaryReader(str);
+            string guid = br.ReadString();
+            Int64 timestamp_ = br.ReadInt64();
+            int len = br.ReadInt32();
+            var payload = br.ReadBytes(len);
+            return new PayloadMessage(guid, timestamp_, payload);
+         }
+      }
+      //public void
+      private readonly int _msg_size;
+      private readonly TimeSpan _publish_period;
+      private DateTime _last_publish_time = DateTime.MinValue;
+      private PublisherCore _pub_core;
+      private ISubscriberCore _sub_core;
+      private BlueTable _table;
+      private int _n_pubs = 0;
+      private readonly int _max_pubs;
+      private int _n_updates = 0;
+      private ViewStreamDatabase _vsd;
+      private PayloadMessage _last_sent_msg, _last_recvd_msg;
+      private const byte _payload_byte = (byte)'A';
+      private readonly string _tag_name;
+      private readonly int _payload_sz;
+      private readonly IPubSubRouter _router;
+      private enum IOState
+      {
+         SEND = 0,
+         RECV = 1
+      };
+      IOState _state = IOState.SEND;
+
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
+      {
+         var route = new PubSubRoute(server_url, ".*");
+         _router = new PubSubRouter(route);
+         _vsd = vsd;
+         _payload_sz = payload_size;
+         var schema = BlueDataUtility.form_schema("PubSubTest",
+         Tuple.Create("id", TypeCode.Int32),
+         Tuple.Create("sent_id", TypeCode.String),
+         Tuple.Create("sent_time", TypeCode.Int64),
+         Tuple.Create("recv_id", TypeCode.String),
+         Tuple.Create("recv_time", TypeCode.Int64),
+         Tuple.Create("match", TypeCode.Boolean),
+         Tuple.Create("echo_time", TypeCode.Int32));
+         _table = new BlueTable(schema);
+         _publish_period = publish_period;
+         _tag_name = tag_name;
+      }
+
+      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
+      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _pub_core = new PublisherCore(_router);
+         _sub_core = new SubscriberCore(_router);
+         _sub_core.subscribe(_tag_name);
+         _vsd.addWorker(this);
+
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         _vsd.removeWorker(this);
+         _sub_core.unsubscribe(_tag_name);
+      }
+
+      private void _fire_dp_after_send()
+      {
+         // as soon as the data is published, its reflected in the stream state
+         Log.assert(_last_sent_msg != null);
+         BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
+            DBNull.Value, DBNull.Value);
+         _table.addRow(r);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+
+      private void _fire_dp_after_recv()
+      {
+         Log.assert(_last_recvd_msg != null);
+         // update the last publish row
+         int id = _table.Count - 1;
+         IBlueRow last_row = _table[id];
+         _table.deleteRow(last_row);
+         var recv_ts = epoch_now();
+         BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
+                                 String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
+         _table.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+
+      private string _get_next_id()
+      {
+         return _table.Count.ToString();
+      }
+
+      private void _publish_next_packet()
+      {
+         var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
+         _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
+         var ser_bytes = _last_sent_msg.serialize();
+         _n_pubs++;
+         _pub_core.publish("Blob", _tag_name, ser_bytes);
+         _last_publish_time = DateTime.Now;
+      }
+
+      public void doWork()
+      {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
+         {
+            foreach (var u in updates)
+            {
+               var recv_msg = PayloadMessage.deserialize(u.data);
+               if (recv_msg.msg_id == _last_sent_msg.msg_id)
+               {
+                  return recv_msg;
+               }
+            }
+            return null;
+         };
+         switch (_state)
+         {
+            case IOState.SEND:
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
+                  _publish_next_packet();
+                  _last_publish_time = DateTime.Now;
+                  _fire_dp_after_send();
+                  _state = IOState.RECV;
+                  return;
+               }
+               break;
+            case IOState.RECV:
+               var updates = _sub_core.get_updates(TimeSpan.Zero);
+               updates = updates.Where(u => u.tag == _tag_name);
+               var recv_msg = find_packet(updates);
+               if (recv_msg != null)
+               {
+                  _last_recvd_msg = recv_msg;
+                  _fire_dp_after_recv();
+                  _state = IOState.SEND;
+                  break;
+               }
+               break;
+         }
+      }
+
+      public static long epoch_now()
+      {
+         return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs

@@ -0,0 +1,58 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+
+   public class SimpleTCPEchoStream : ViewStream, IWorker {
+      private string _ip;
+      private int _port;
+      private SimpleTCPClient _tcp_client;
+      private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
+      Tuple.Create("id", TypeCode.Int32),
+      Tuple.Create("event", TypeCode.String));
+      private BlueTable _table;
+      private SyncQueue<string> _action_queue = new SyncQueue<string>();
+
+      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
+      : this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
+
+      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
+         _ip = ip;
+         _port = port;
+         _table = new BlueTable(_schema);
+         _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         ViewStreamDatabase.instance.addWorker(this);
+         _tcp_client.register_listener((msg) => { _on_msg(msg); });
+         _tcp_client.start();
+      }
+
+      private void _on_msg(string msg) {
+         _action_queue.enqueue(msg);
+      }
+
+      public void doWork() {
+         if (_action_queue.getCount() == 0)
+            return;
+         BlueTable added_rows = new BlueTable(_schema);
+         var sz = _action_queue.getCount();
+         for (int i = 0; i < sz; i++) {
+            string msg = _action_queue.dequeue();
+            if (msg == null)
+               break;
+            IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
+            _table.addRow(nr);
+            added_rows.addRow(nr);
+         }
+         if (added_rows.Count == 0)
+            return;
+         DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
+         myFireEvent(dp);
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs

@@ -0,0 +1,77 @@
+using System.Net.Sockets;
+using System.Net;
+
+using BCG;
+using System;
+using System.Collections.Generic;
+using MongoDB.Bson;
+using BCG.Utilities.General;
+using System.Linq;
+
+
+namespace bts {
+   // this class, a little specialized, takes the first snapshot thats fired 
+   // from the base stream sorts them by a given column, and fires one row at a time 
+   public class SnapshotReplayStream : ViewStream, IWorker {
+
+      private ViewStreamDatabase _vsd;
+      private ViewStream _snapshot_stream;
+      private readonly string _sort_col;
+      private readonly int _buffer_size;
+      private TimeSpan _fire_period;
+      private int _in_row_ptr = 0;
+      private List<IBlueRow> _in_rows = new List<IBlueRow>();
+      private CircularBufferTable _out_table;
+
+
+      public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period)
+      : this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { }
+
+      public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) {
+         _vsd = vsd;
+         _snapshot_stream = base_str;
+         _sort_col = sort_col;
+         _buffer_size = buffer_size;
+         _fire_period = fire_period;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _snapshot_stream.onChange += _handle_stream_update;
+         MasterWorker.getInstance(false).addWorker(this, _fire_period);
+      }
+
+      protected override void myOnLastSubscriber() {
+         base.myOnLastSubscriber();
+      }
+
+      private void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         if (_in_rows.Count > 0)
+            return;
+         _out_table = new CircularBufferTable(dp.schema, _buffer_size);
+         // _in_rows is filled up only once
+         foreach (IBlueRow r in dp.newRows) {
+            _in_rows.Add(r);
+         }
+         _in_rows.Sort((IBlueRow r1, IBlueRow r2) => {
+            return BlueDataUtility.compare_rows(r1, r2, _sort_col);
+         });
+      }
+
+      public void doWork() {
+         if (_in_row_ptr >= _in_rows.Count) {
+            // we can loop around or not here
+            return;
+         }
+         if (_in_rows.Count > 0) {
+            // fire row
+            IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
+            _out_table.add_row(curr_row);
+            DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
+               myLastValue == null ? null : myLastValue as DataPacket, false);
+            myFireEvent(dp);
+         }
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -0,0 +1,809 @@
+using bts;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+
+namespace BCG {
+
+    /// <summary>
+    /// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
+    /// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
+    /// </summary>
+    [DataStreamUserRespresentation("tests")]
+    public class TesStream : ViewStream {
+        static byte[] create_byte_array() {
+            Random random = new Random();
+            var sz = random.Next(1, 1000);// % 1000;
+
+            // Generate a random byte
+            byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
+            byte[] rv = new byte[sz];
+            for (int i = 0; i < sz; i++) {
+                rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
+            }
+            return rv;
+        }
+
+        private IBlueTableSchema _schema;
+        class TestResult {
+            public string test_name = MethodBase.GetCurrentMethod()?.Name
+               ?? throw new InvalidOperationException("Failed to get test name from method");
+            public bool test_result;
+        }
+        private List<Func<TestResult>> _tests = new
+         List<Func<TestResult>>();
+
+        public TesStream() {
+            _schema = BlueDataUtility.form_schema("Tests",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("test", TypeCode.String),
+            Tuple.Create("result", TypeCode.Boolean),
+            Tuple.Create("exception", TypeCode.String)
+            );
+            // register tests here
+            _tests.Add(new Func<TestResult>(_run_sub_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pub_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
+            _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
+            // _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
+            // _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
+            //  _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
+            //_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
+            // _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
+            // _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
+            // _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
+            // _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
+            //_tests.Add(new Func<TestResult>(_cpp_data_test));
+            //_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
+            _tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
+
+        }
+
+        protected override void myOnFirstSubscriber() {
+            BlueTable rows = new BlueTable(_schema);
+            // u the tests here            
+            foreach (var test in _tests) {
+                string ex_msg = String.Empty;
+                TestResult tr = new TestResult() {
+                    test_name = test.Method.Name,
+                    test_result = false
+                };
+                try {
+                    tr = test();
+                } catch (Exception ex) {
+                    ex_msg = ex.Message + "\n" + ex.StackTrace;
+                }
+                IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
+                rows.addRow(r);
+            }
+            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+        protected override void myOnLastSubscriber() {
+        }
+
+        private TestResult _run_sub_msg_test() {
+            bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
+                connection = Guid.NewGuid().ToString(),
+                subscribing_app = Guid.NewGuid().ToString(),
+                tags = new string[] { "one", "two", "three" }
+            };
+            MemoryStream str = new MemoryStream();
+            sub_msg.serialize_into_str(str);
+            str.Position = 0;
+            bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
+            sub_msg_.consruct_deserialize(str);
+            bool res = sub_msg.Equals(sub_msg_);
+            return new TestResult {
+                test_name = "SubscribeMsg",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pub_msg_test() {
+            var pub_msg = new bts.PublishMsg() {
+                connection = Guid.NewGuid().ToString(),
+                publishing_app = Guid.NewGuid().ToString(),
+                tag = Guid.NewGuid().ToString(),
+                tag_type = "DS_",
+                is_compressed = false,
+                publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                data_to_publish = create_byte_array(),
+            };
+            MemoryStream str = new MemoryStream();
+            pub_msg.serialize_into_str(str);
+            str.Position = 0;
+            var pub_msg_ = new bts.PublishMsg();
+            pub_msg_.consruct_deserialize(str);
+            bool res = pub_msg.Equals(pub_msg_);
+            return new TestResult {
+                test_name = "PublishMsg",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pubsub_server_msg_test() {
+            var msg = new bts.PubSubServerMessage() {
+                tag = Guid.NewGuid().ToString(),
+                tag_type = Guid.NewGuid().ToString(),
+                data = create_byte_array()
+            };
+            MemoryStream str = new MemoryStream();
+            msg.serialize_into_str(str);
+            str.Position = 0;
+            var msg_ = new bts.PubSubServerMessage();
+            msg_.consruct_deserialize(str);
+            bool res = msg.Equals(msg_);
+            return new TestResult {
+                test_name = "PubSubServerMessage",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pubsub_client_msg_test() {
+            PubSubClientMessage msg = new PubSubClientMessage() {
+                msg_type = PubSubMessageType.SUBSCRIBE,
+                sub_msg = new bts.SubscribeMsg() {
+                    connection = Guid.NewGuid().ToString(),
+                    subscribing_app = Guid.NewGuid().ToString(),
+                    tags = new string[] { "one", "two", "three" }
+                }
+            };
+            var str = new MemoryStream();
+            msg.serialize_into_str(str);
+            str.Position = 0;
+            var msg_ = new PubSubClientMessage();
+            msg_.consruct_deserialize(str);
+            var res = msg.Equals(msg_);
+            return new TestResult {
+                test_name = "PubSubClientMessage",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_columninfo_serialization_test() {
+            Func<ColumnInfo, bool> test_ci = (ci) => {
+                MemoryStream str = new MemoryStream();
+                BinaryWriter bw = new BinaryWriter(str);
+                StreamSerializationUtils.serialize_column_info(ci, ref bw);
+                str.Position = 0;
+                //var ci_txt = BlueDataUtility.to_string(ci);
+                //var ci_ser = 
+                //bts.Log.Gen.info("ci: {}")
+                BinaryReader br = new BinaryReader(str);
+                var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
+                return BlueDataUtility.compute_equality(ci, ci_prime);
+            };
+            int i = 0;
+            foreach (Type t in BlueDataUtility.valid_column_types()) {
+                ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
+                i++, t);
+                var res = test_ci(ci);
+                if (!res) {
+                    return new TestResult {
+                        test_name = "ColumnInfoSerialization",
+                        test_result = false
+                    };
+                }
+            }
+            return new TestResult {
+                test_name = "ColumnInfoSerialization",
+                test_result = true
+            };
+        }
+
+
+        private TestResult _run_schema_serialization_test() {
+            var table_name = "test_table";
+            var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
+            IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            StreamSerializationUtils.serialize_schema(schema, ref bw);
+            _write_to_file(str, @"c:\Temp\tests\schema.bin");
+            var hex_txt = StreamSerializationUtils.to_hex(str);
+            bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
+
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
+            var res = BlueDataUtility.compute_equality(schema, schema_prime);
+
+            return new TestResult {
+                test_name = "SchemaSerialization",
+                test_result = res
+            };
+        }
+
+
+
+
+        private TestResult _run_datetime_seriliaztion_test() {
+            DateTime dt = DateTime.Now;
+            DateTimeOffset dt_off = new DateTimeOffset(dt);
+            long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
+            DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
+            DateTime dt_p = dt_off_pr.DateTime;
+            var passes = dt.Equals(dt_p);
+            return new TestResult {
+                test_name = "_run_datetime_seriliaztion_test",
+                test_result = false
+            };
+        }
+
+        private TestResult _run_full_stream_serialization_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
+            UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
+            ser.serialize(stream_io_msg, ref bw);
+            str.Position = 0;
+            BinaryReader br = new System.IO.BinaryReader(str);
+            UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
+            var stream_io_msg_prime = deser.deserialize(ref br);
+            bool passes = stream_io_msg.Equals(stream_io_msg_prime);
+            var n_dels = 50;
+            BlueTable del_rows = new BlueTable(cr_rows.schema);
+            foreach (IBlueRow r in cr_rows) {
+                del_rows.addRow(r);
+                if (del_rows.Count == n_dels)
+                    break;
+            }
+            IBlueRowCollection added_rows = del_rows;
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            str = new MemoryStream();
+            bw = new BinaryWriter(str);
+            stream_io_msg = new bts.StreamIOMessage(mra_msg);
+            ser.serialize(stream_io_msg, ref bw);
+            bw.BaseStream.Position = 0;
+            br = new BinaryReader(str);
+            //deser = new UniversalStreamIODeserializer();
+            stream_io_msg_prime = deser.deserialize(ref br);
+            passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
+            return new TestResult {
+                test_name = "UniversalSerialization",
+                test_result = passes
+            };
+        }
+
+        private TestResult _run_full_stream_serialization_with_controller_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
+            IStreamIOMsgSerializer ser = new StreamIOSerializationController();
+            var ser_bytes = ser.serialize(stream_io_msg);
+            IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
+            var stream_io_msg_prime = deser.deserialize(ser_bytes);
+            bool passes = stream_io_msg.Equals(stream_io_msg_prime);
+            //mult remove adds
+            // remove all rows and add 50 more
+
+            BlueTable del_rows = new BlueTable(cr_rows);
+            BlueTable added_rows = new BlueTable(cr_rows.schema);
+            var n_adds = 50;
+            for (int i = 0; i < n_adds; i++) {
+                var nr = generate_random_row(schema, i);
+                added_rows.addRow(nr);
+            }
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            stream_io_msg = new bts.StreamIOMessage(mra_msg);
+            ser_bytes = ser.serialize(stream_io_msg);
+            var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
+            Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
+            stream_io_msg_prime = deser.deserialize(ser_bytes);
+            passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
+            return new TestResult {
+                test_name = "UniversalSerializationController",
+                test_result = passes
+            };
+        }
+
+        private TestResult _row_serialization_tests_no_dbnulls() {
+            // IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+            //                                                     new Tuple<string, TypeCode>("id", TypeCode.Int32),
+            //                                                     new Tuple<string, TypeCode>("i1", TypeCode.Int32),
+            //                                                     new Tuple<string, TypeCode>("d1", TypeCode.Double),
+            //                                                     new Tuple<string, TypeCode>("string_1", TypeCode.String));
+            // bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            // IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
+            var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            MemoryStream str = new MemoryStream();
+            IBlueRow r = generate_random_row(schema, 0, false);
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            _write_to_file(str, @"C:\Temp\tests\row.bin");
+            str.Position = 0;
+            byte[] raw_byte_array = str.ToArray();
+            var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
+            Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            string row_str = r_prime.to_string();
+            Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_NonNull",
+                test_result = passes
+            };
+        }
+
+        private TestResult _deserialize_pub_msg_test() {
+            var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
+            PubSubClientMessage ps_client_msg = new PubSubClientMessage();
+            ps_client_msg.consruct_deserialize(pub_msg_ser);
+            bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
+            return new TestResult {
+                test_name = "_deserialize_pub_msg_test",
+                test_result = passes
+            };
+        }
+
+        private TestResult _cpp_data_test() {
+            //create and write schema
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d_1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String)
+                                                                );
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
+            StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+            StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+
+            IBlueTableSchema schema_p = null;
+            var schema_file = @"c:\Temp\tests\schema.bin";
+            var row_file = @"C:\Temp\tests\row.bin";
+            var create_op_file = @"C:\Temp\tests\create_op.bin";
+            var mra_op_file = @"C:\Temp\tests\mra_op.bin";
+            IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
+
+            //serialize schema, write it to a file, read it back, deserialize it and match schemas
+            byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
+            _write_to_file(ser_schema_bytes, schema_file);
+
+            byte[] schema_file_bytes = _read_from_file(schema_file);
+            schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
+            bool passes = BlueDataUtility.compute_equality(schema, schema_p);
+
+            //serialize row, write it to a file, read it back, deserialize it and match schemas    
+            byte[] ser_row_bytes = row_ser.serialize_row(row);
+            _write_to_file(ser_row_bytes, row_file);
+
+            byte[] row_file_bytes = _read_from_file(row_file);
+            var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
+            Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
+
+            IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
+            passes = passes && BlueDataUtility.compute_equality(row, row_p);
+
+            {
+                // serialize a create operation
+                int n_rows = 100;
+                BlueTable table = new BlueTable(schema);
+                for (int i = 0; i < n_rows; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
+                //StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
+                _write_to_file(create_ser_bytes, create_op_file);
+
+                // read it and make sure its right
+                byte[] create_op_file_bytes = _read_from_file(create_op_file);
+                //StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+                var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
+                _write_to_file(create_ser_bytes, mra_op_file);
+
+                passes = passes && cr_msg.Equals(cr_msg_p);
+
+            }
+            //multiple remove adds
+            {
+                // remove all rows and add them again
+                int n_rows = 100;
+                BlueTable table = new BlueTable(schema);
+                for (int i = 0; i < n_rows; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
+
+                //StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
+                _write_to_file(mra_ser_bytes, mra_op_file);
+
+                // reasd it and make sure it matches
+                byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
+                //StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+                StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
+                passes = passes && mra_msg.Equals(mra_msg_p);
+            }
+            return new TestResult {
+                test_name = "_cpp_data_test",
+                test_result = passes
+            };
+        }
+
+        private TestResult _write_stream_updates_to_files() {
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d_1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String)
+                                                                );
+            var packets_path = @"C:\Temp\tests\stream_packets\";
+            Func<long> now_epochs = () => {
+                DateTimeOffset now = DateTimeOffset.UtcNow;
+                // Convert to Unix time in milliseconds
+                long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
+                return unix_time_milliseconds;
+            };
+            Func<int, bool, string> curr_file_path = (id, create) => {
+                //var epoch_ms = now_epochs();
+                var file_path = $"{packets_path}{id}.bin";
+                if (create)
+                    file_path = $"{packets_path}create_{id}.bin";
+                return file_path;
+            };
+            // write 1000 stream updates
+            {
+                int init_sz = 100;
+                BlueTable table = new BlueTable(schema);
+                StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                // create table of 100 rows
+                for (int i = 0; i < init_sz; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                // write create msg
+                StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
+                byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
+                var create_file_path = curr_file_path(table.Count, true);
+                _write_to_file(create_ser_bytes, create_file_path);
+
+                // add 1000 rows and write an io msg for each update and write it to a file
+                int n_updates = 1000;
+                int period_ms = 50;
+
+                // write a create
+                for (int i = 0; i < n_updates; i++) {
+                    // addition
+                    BlueTable adds = new BlueTable(schema);
+                    int id = table.Count;
+                    IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
+                    adds.addRow(nr);
+                    BlueTable dels = new BlueTable(schema);
+                    table.addRow(nr);
+                    StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
+                    byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
+                    var mra_file_path = curr_file_path(id, false);
+                    _write_to_file(mra_ser_bytes, mra_file_path);
+
+                    System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
+                }
+                // now start writing increments and add 1 row at a time
+            }
+            return new TestResult {
+                test_name = "_write_updates_as_files",
+                test_result = true
+            };
+        }
+
+
+        private TestResult _row_serialization_tests_with_dbnulls() {
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String));
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_WithDBNulls",
+                test_result = passes
+            };
+        }
+
+        private TestResult _row_serialization_tests_all_types() {
+            var schema = create_schema_with_all_types();
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            IBlueRow r = generate_random_row(schema, 0, false);
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            _write_to_file(str, @"C:\Temp\tests\row.bin");
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_AllTypes",
+                test_result = passes
+            };
+        }
+
+        private static TestResult _run_pubsub_server_stream_tag_handler_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            var tag = "DS_" + schema.tableName;
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+
+            PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
+            IStreamIOMsgSerializer ser = new StreamIOSerializationController();
+            byte[] msg_bytes = ser.serialize(str_io_msg);
+            handler.update(tag, msg_bytes);
+
+            //now get it back for the handler
+            byte[] content = handler.get_content(tag);
+            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
+            bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
+            // test here
+            var passes = str_io_msg.Equals(str_io_msg_p);
+            // delete all rows
+            BlueTable del_rows = new BlueTable(cr_rows);
+            cr_rows.clear();
+            // added 50 new rows
+            BlueTable added_rows = new BlueTable(del_rows.schema);
+            int new_state_sz = 50;
+            for (int i = 0; i < new_state_sz; i++) {
+                var nr = generate_random_row(schema, i);
+                added_rows.addRow(nr);
+            }
+            cr_rows.addRows(added_rows);
+            // the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
+            // 1. generate op
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            str_io_msg = new bts.StreamIOMessage(mra);
+            // 2. serialize it into bytes
+            msg_bytes = ser.serialize(str_io_msg);
+            // 3. send it to handler
+            handler.update(tag, msg_bytes);
+            // 4. get the state from the handler
+            byte[] content_p = handler.get_content(tag);
+            // 5. deserialize the state into a stream op
+            str_io_msg_p = deser.deserialize(content_p);
+            // 6. check to make sure the state's rows are good
+            passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
+                    BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
+            return new TestResult {
+                test_result = passes,
+                test_name = "run_pubsub_server_stream_tag_handler_test"
+            };
+        }
+
+
+        private static TestResult _run_single_tag_data_test_clean() {
+            // create message
+            var test_name = MethodBase.GetCurrentMethod().Name;
+            var schema = create_schema_with_all_types();
+            var tag = "DS_" + schema.tableName;
+            BlueTable table = new BlueTable(schema);
+            IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
+            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
+            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
+            // create with 100 rows
+            int n_rows = 100;
+            bool is_succ = true;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                // perform the same action SingletagData, and verify the states match
+                PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
+                table.addRow(nr);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            // change all rows
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                var dr = table[i];
+                table.deleteRow(dr);
+                table.addRow(nr);
+                PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            // delete all rows
+            var aux_table = new BlueTable(table);
+            foreach (IBlueRow r in aux_table) {
+                table.deleteRow(r);
+                PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            return new TestResult() {
+                test_result = is_succ,
+                test_name = test_name
+            };
+        }
+
+
+
+        #region Helper functions
+
+        private static void _write_to_file(byte[] ser_bytes, string file_name) {
+            _write_to_file(new MemoryStream(ser_bytes), file_name);
+        }
+
+        private static void _write_to_file(MemoryStream str, string file_name) {
+            if (String.IsNullOrEmpty(file_name))
+                return;
+            byte[] str_bytes = str.ToArray();
+            using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
+                // Write the byte array to the file
+                fs.Write(str_bytes, 0, str_bytes.Length);
+            }
+        }
+
+        private static byte[] _read_from_file(string file_path) {
+            var rv = File.ReadAllBytes(file_path);
+            return rv;
+        }
+
+        public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
+            var content = st_data.get_current_content();
+            var msg_p = deser.deserialize(content);
+            Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
+            return msg_p.create_message.rows;
+
+        }
+
+        public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
+            ref IStreamIOMsgSerializer serializer) {
+            Func<IBlueTableSchema> get_schema = () => {
+                return (added_row != null ? added_row : removed_row).schema;
+            };
+            bts.StreamIOMessage str_io_msg = null;
+            if (is_create) {
+                Log.assert(added_row != null);
+                var aux_new = new BlueTable(added_row.schema);
+                aux_new.addRow(added_row);
+                var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
+                str_io_msg = new bts.StreamIOMessage(cr_msg);
+            } else {
+                var schema = get_schema();
+                BlueTable added_table = new BlueTable(schema);
+                if (added_row != null)
+                    added_table.addRow(added_row);
+                BlueTable rem_table = new BlueTable(schema);
+                if (removed_row != null)
+                    rem_table.addRow(removed_row);
+                bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
+                    new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
+                str_io_msg = new bts.StreamIOMessage(mra_msg);
+            }
+            byte[] msg_bytes = serializer.serialize(str_io_msg);
+            PublishMsg pub_msg = new PublishMsg() {
+                connection = "test_connection",
+                tag = $"DS_{get_schema()}",
+                tag_type = "DS",
+                data_to_publish = msg_bytes
+            };
+            return pub_msg;
+        }
+
+        public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
+            excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
+            var table_name = "test_table";
+            BlueTableSchema schema = new BlueTableSchema();
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            foreach (Type t in BlueDataUtility.valid_column_types()) {
+                if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
+                    continue;
+                var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
+                ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
+                cis.Add(ci);
+            }
+            cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
+            schema = new BlueTableSchema(table_name, cis.ToArray());
+            return schema;
+        }
+
+        public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
+            Random rnd = new Random();
+            object[] row_vals = new object[schema.columnInfo.Length];
+            foreach (ColumnInfo ci in schema.columnInfo) {
+                if (ci.colName == "id") {
+                    row_vals[ci.colNo] = id;
+                    continue;
+                }
+                object v = get_random_object(ci.type_code());
+                if (include_dbnulls && rnd.NextSingle() % 4 == 0)
+                    v = DBNull.Value;
+                row_vals[ci.colNo] = v;
+            }
+            return new BlueRow(schema, row_vals);
+        }
+
+        public static object get_random_object(TypeCode type_code) {
+            Random random = new Random();
+            switch (type_code) {
+                case TypeCode.Boolean:
+                    return random.Next(2) == 0; // Random boolean value (true or false)
+                case TypeCode.Byte:
+                    return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
+                case TypeCode.Char:
+                    return (char)random.Next(65, 65 + 27); // Random char value
+                case TypeCode.Decimal:
+                    Decimal dec = 100.123M;
+                    return dec;
+                case TypeCode.DateTime:
+                    return DateTime.Now;
+                case TypeCode.Double:
+                    return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
+                case TypeCode.Int16:
+                    return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
+                case TypeCode.Int32:
+                    return random.Next(int.MinValue, int.MaxValue); // Random int value
+                case TypeCode.Int64:
+                    return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
+                case TypeCode.SByte:
+                    return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
+                case TypeCode.Single:
+                    return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
+                case TypeCode.String:
+                    return Guid.NewGuid().ToString();
+                case TypeCode.UInt16:
+                    return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
+                case TypeCode.UInt32:
+                    return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
+                case TypeCode.UInt64:
+                    byte[] ulongBuffer = new byte[8];
+                    random.NextBytes(ulongBuffer);
+                    return BitConverter.ToUInt64(ulongBuffer, 0);
+                default:
+                    throw new ArgumentException($"Unsupported TypeCode: {type_code}");
+                    return null;
+            }
+        }
+        #endregion
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TickingTestStream.cs

@@ -0,0 +1,31 @@
+namespace BCG {
+   using System;
+
+   public class TickingTestStream : ViewStream {
+      public TickingTestStream(int ticking_period_msecs)
+      :
+      this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
+
+      public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
+         ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
+         vs = new MathStream(vs, "x=id*123 y=x*x");
+         mySetRealSource(vs);
+         //vs.onChange += _on_str_update;
+      }
+
+      // private void _on_str_update(object dp_obj) {
+      //    DataPacket dp = dp_obj as DataPacket;
+      //    IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
+      //    BlueTable state = new BlueTable(out_schema);
+      //    foreach (IBlueRow r in dp.newRows) {
+      //       MutableBlueRow nr = new MutableBlueRow(out_schema, r);
+      //       DateTime time = r.get_datetime("time");
+      //       var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+      //       nr.setValue("time_txt", time_txt);
+      //       state.addRow(nr);
+      //    }
+      //    var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
+      //    myFireEvent(dp_out);
+      // }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -1,9 +1,8 @@
-using Google.Protobuf.WellKnownTypes;
-
 using System;
-using BCG.Utilities.General;
 
-namespace BCG {
+using BCG;
+
+namespace bts.SilverPine {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 
@@ -41,7 +40,6 @@ namespace BCG {
          }
          _table.addRows(added_rows);
          myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
-
       }
    }
 

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs

@@ -0,0 +1,194 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using Bts.DataStreamCore;
+
+using mbts;
+
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Reactive.Subjects;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
+   readonly string collectionName;
+   readonly string databaseName;
+   readonly string url;
+   readonly object syncRoot = new();
+   readonly CancellationTokenSource cts = new();
+   readonly Subject<DataPacket> subject = new();
+
+   bool hasStarted = false;
+   IBlueRowFactory<BsonDocument>? rowFactory;
+   MongoClient? clientBacking;
+   IMongoDatabase? databaseBacking;
+   IMongoCollection<BsonDocument>? collectionBacking = null;
+   BlueTableImmutable? table = null;
+
+   public MongoCollectionStream(string collectionName)
+      : this(
+         collectionName,
+         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName, string databaseName)
+      : this(
+         collectionName,
+         databaseName,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName, string databaseName, string url)
+      : this(collectionName, databaseName, url, null) { }
+
+   internal MongoCollectionStream(
+      string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
+      this.collectionName = collectionName;
+      this.databaseName = databaseName;
+      this.url = url;
+      this.rowFactory = rowFactory;
+      mySetRealSource(new ObservableViewStream(this));
+   }
+
+   MongoClient client => clientBacking ??= new(url);
+   IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
+   IMongoCollection<BsonDocument>? collection {
+      get => Volatile.Read(ref collectionBacking);
+      set => Volatile.Write(ref collectionBacking, value);
+   }
+
+   public IDisposable Subscribe(IObserver<DataPacket> observer) {
+      lock (syncRoot) {
+         if (hasStarted) {
+            if (table is not null) {
+               // The first update has already been fired. Seed observer w/ current state.
+               observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
+            }
+            return subject.Subscribe(observer);
+         }
+         hasStarted = true;
+      }
+      var subscription = subject.Subscribe(observer);
+      // Retrieve collection snapshot to bootstrap sequence
+      initTable().ignore();
+      // Watch changes
+      Task.Run(watchChanges);
+      return Disposable.Create(Dispose);
+
+      void Dispose() {
+         Log.Gen.info("Disposing MongoDbChangeObservable subscription");
+         subscription.Dispose();
+         cts.Cancel();
+         cts.Dispose();
+      }
+   }
+
+   protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
+      MongoSchemaAdaptor.InferSchema(docs);
+
+   async Task watchChanges() {
+      Log.Gen.info(
+         $"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}");
+      while (collection is null) {
+         await Task.Delay(100);
+      }
+      Log.notnull(collection);
+      using var cursor = collection.Watch(new ChangeStreamOptions {
+         FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
+      });
+      try {
+         while (await cursor.MoveNextAsync(cts.Token)) {
+            /// Ignore while waiting for init.
+            lock (syncRoot) {
+               /// Note: <see cref="rowFactory"/> and <see cref="table"/> are initialized together
+               /// in the <see cref="initTable"/> fn since both depend on the schema, which is
+               /// inferred from the first incoming BsonDocument(s).
+               if (table is null || rowFactory is null) {
+                  continue;
+               }
+            }
+
+            var bd = table.ToDataPacketBuilder();
+            foreach (var ch in cursor.Current) {
+               var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
+               var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
+               Log.Gen.info(
+                  $"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
+                  $"changes:\n\t\t{updDescJson}");
+               switch (ch.OperationType) {
+                  case ChangeStreamOperationType.Insert: {
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Update: {
+                        bd.Remove(OldRow().Id());
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Replace: {
+                        bd.Remove(OldRow().Id());
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Delete: {
+                        bd.Remove(OldRow().Id());
+                        break;
+                     }
+                  default:
+                     Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
+                     break;
+               }
+
+               IBlueRow Row() =>
+                  rowFactory.GetBlueRow(ch.FullDocument);
+
+               IBlueRow OldRow() =>
+                  rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
+            }
+            var dp = bd.ToDataPacket();
+            if (!dp.HasChanges()) {
+               Log.Gen.warn(
+                  $"MongoCollectionStream. Ignoring update because it didn't change any rows.");
+               continue;
+            }
+            lock (syncRoot) {
+               table = (BlueTableImmutable)dp.newRows;
+               subject.OnNext(dp);
+            }
+         }
+      } catch (TaskCanceledException ex) {
+         Log.Gen.warn(
+            $"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}");
+      }
+   }
+
+
+   [MemberNotNull(nameof(collection))]
+   async Task initTable() {
+      collection = database.GetCollection<BsonDocument>(collectionName);
+      var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+      var docList = await cursor.ToListAsync();
+      rowFactory ??= getRowFactory(docList);
+      var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
+      foreach (var doc in docList) {
+         var row = rowFactory.GetBlueRow(doc);
+         firstTable.AddOrUpdate(row);
+      }
+      var dp = firstTable.ToDataPacket();
+      lock (syncRoot) {
+         table = (BlueTableImmutable)dp.newRows;
+         subject.OnNext(dp);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,128 +1,133 @@
-using System;
-using System.Collections.Generic;
-using MongoDB.Driver;
-using MongoDB.Bson;
 using BCG;
 using BCG.Utilities.General;
-using ZstdSharp.Unsafe;
-using System.Runtime.CompilerServices;
-using mbts.spark;
+
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Threading;
+
+namespace bts {
+    // [DataStreamUserRespresentation("mongodb_sql")]
+    // public class MongoDBSQLStream : ViewStream, IWorker {
+    //     private ViewStreamDatabase _vsd;
+    //     private readonly string _connection_string, _db_name, _collection_name;
+    //     private MongoClient _client;
+    //     // make this a sorted table
+    //     private BlueTable _table;
 
 
-namespace BCG {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
+    //     // Access the database
+    //     private IMongoDatabase _database;
+    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+    //     // mongodb://192.168.1.80:27017
 
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+    //     }
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+    //                                 $"db_name: {db_name}");
+    //         _connection_string = connection_string;
+    //         _db_name = db_name;
+    //         _collection_name = collection_name;
+    //         _vsd = vsd;
+    //         _client = new MongoClient(_connection_string);
+    //         _database = _client.GetDatabase(_db_name);
+    //     }
 
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+    //     protected override void myOnFirstSubscriber() {
+    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+    //     }
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
+    //     protected override void myOnLastSubscriber() {
+    //         MasterWorker.getInstance(false).removeWorker(this);
+    //     }
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+    //     public void doWork() {
+    //         _read_mogodb();
+    //     }
 
-        public void doWork() {
-            _read_mogodb();
-        }
+    //     private void _read_mogodb() {
+    //         var timer = Stopwatch.StartNew();
+    //         // Access the collection
+    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+    //         // Fetch all documents from the collection
+    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+    //         _table = _table == null ? null : new BlueTable(_table.schema);
+    //         // Iterate through the documents and print values
+    //         foreach (var document in documents.ToEnumerable()) {
+    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+    //             if (_table == null) {
+    //                 _init_table(document);
+    //             }
+    //             _add_row(document);
+    //         }
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+    //         timer.Restart();
+    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+    //             return;
+    //         myFireEvent(dp);
+    //     }
 
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+    //     private void _init_table(BsonDocument doc_row) {
+    //         bts.Log.assert(_table == null);
+    //         List<ColumnInfo> cis = new List<ColumnInfo>();
+    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+    //         foreach (var n in doc_row.Names) {
+    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+    //         }
+    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+    //         _table = new BlueTable(schema);
+    //     }
 
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
-
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
+    //     private void _add_row(BsonDocument doc) {
+    //         var key_val = doc["_id"];
+    //         var key = key_val.ToString();
+    //         int id;
+    //         if (!_id_map.TryGetValue(key, out id)) {
+    //             id = _id_map.Count;
+    //             _id_map.Add(key, id);
+    //         }
+    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
+    //         nr.setValue("id", id);
+    //         foreach (var n in doc.Names) {
+    //             var val = doc[n].ToString();
+    //             nr.setValue(n, val);
+    //         }
+    //         _table.addRow(nr);
+    //     }
+    // }
 
     [DataStreamUserRespresentation("mongodb")]
     public class MongoDBStream : ViewStream, IWorker {
         private ViewStreamDatabase _vsd;
         private readonly string _connection_string, _db_name, _collection_name;
+        private readonly TimeSpan _poll_period;
         private MongoClient _client;
         private BlueTable _table;
+        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
+        private BlockingQueue _work_queue = new BlockingQueue();
 
         // Access the database
         private IMongoDatabase _database;
         private IDictionary<string, int> _id_map = new Dictionary<string, int>();
         // mongodb://192.168.1.80:27017
 
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
+        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
 
         }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
             bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                     $"db_name: {db_name}");
             _connection_string = connection_string;
@@ -130,24 +135,34 @@ namespace BCG {
             _collection_name = collection_name;
             _vsd = vsd;
             _client = new MongoClient(_connection_string);
+            _poll_period = poll_period;
             _database = _client.GetDatabase(_db_name);
         }
 
         protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+            _vsd.addWorker(this);
+            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
         }
 
         protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
+            _vsd.removeWorker(this);
+            //MasterWorker.getInstance(false).removeWorker(this);
         }
 
+        private DateTime _last_poll_time = DateTime.MinValue;
         public void doWork() {
-            _read_mogodb();
+            var now = DateTime.Now;
+            if (now - _last_poll_time > _poll_period) {
+                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
+                _last_poll_time = DateTime.Now;
+            }
+            var dp = _table_queue.dequeue(0);
+            if (dp != null)
+                myFireEvent(dp);
         }
 
         private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
+            var timer = Stopwatch.StartNew();
             // Access the collection
             IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
@@ -156,21 +171,22 @@ namespace BCG {
             _table = _table == null ? null : new BlueTable(_table.schema);
             // Iterate through the documents and print values
             foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                //bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
                 if (_table == null) {
                     _init_table(document);
                 }
                 _add_doc(document);
             }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
+            timer.Stop();
+            Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
+            timer.Restart();
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            timer.Stop();
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
             if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                 return;
-            myFireEvent(dp);
+            _table_queue.enqueue(dp);
+            //myFireEvent(dp);
         }
 
         private void _init_table(BsonDocument doc_row) {
@@ -209,4 +225,46 @@ namespace BCG {
         }
     }
 
-}
+    // [DataStreamUserRespresentation("mongodb_flat")]
+    // public class MongoDBFlatStream : ViewStream {
+    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+    //     :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+
+    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+    //         mySetRealSource(rs);
+    //     }
+    // }
+
+
+
+    // [DataStreamUserRespresentation("newsreplay")]
+    // public class NewsReplayStream : ViewStream {
+    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, string fire_period) :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+
+    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, TimeSpan fire_period) {
+    //         ViewStream r;
+
+    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         r = new KeyValueToRelationalStream(r,
+    //                                             "doc_id",
+    //                                             "key",
+    //                                             "value");
+    //         r = new DateStandardizationStream(r,
+    //                                            "created_at",
+    //                                            "ISO8601");
+    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+
+    //         mySetRealSource(r);
+    //     }
+    // }
+
+
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs

@@ -0,0 +1,63 @@
+namespace Bts.DataStream;
+
+using MongoDB.Bson;
+
+using System;
+
+public static class MongoDbExtensions {
+   public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsBsonArray;
+
+   public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
+         val.AsBsonArray : null;
+
+   public static string GetString(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsString;
+
+   public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsString ?
+         val.AsString : null;
+
+   public static int GetInt32(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsInt32;
+
+   public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
+         val.AsInt32 : null;
+
+   public static long GetInt64(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsInt64;
+
+   public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
+         val.AsInt64 : null;
+
+   public static double GetDouble(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsDouble;
+
+   public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsDouble ?
+         val.AsDouble : null;
+
+   public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
+      doc[elementName].ToUniversalTime();
+
+   public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
+         val.ToUniversalTime() : null;
+
+   public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
+      doc[elementName].ToUniversalTime();
+
+   public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
+         val.ToUniversalTime() : null;
+
+   public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsObjectId;
+
+   public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
+         val.AsObjectId : null;
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs

@@ -0,0 +1,55 @@
+namespace Bts.DataStream;
+
+using bts;
+
+using MongoDB.Bson;
+
+using System;
+using System.Collections.Generic;
+
+public static class MongoSchemaAdaptor {
+   public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
+      var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
+      foreach (var doc in docs) {
+         foreach (var el in doc) {
+            if (schemaAdaptor.Contains(el.Name)) {
+               continue;
+            }
+            switch (el.Value.BsonType) {
+               case BsonType.Array:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
+                  break;
+               case BsonType.String:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
+                  break;
+               case BsonType.Int32:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
+                  break;
+               case BsonType.Int64:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
+                  break;
+               case BsonType.Double:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
+                  break;
+               case BsonType.DateTime:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
+                  break;
+               case BsonType.Timestamp:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
+                  break;
+               case BsonType.ObjectId:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
+                  break;
+               case BsonType.Null:
+                  Log.Gen.warn(
+                     $"Ignoring column '{el.Name}' because at least one element is null");
+                  break;
+               default:
+                  throw new InvalidOperationException(
+                     $"Unsupported BsonType '{el.Value.BsonType}'");
+            }
+         }
+      }
+      return schemaAdaptor;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -0,0 +1,202 @@
+namespace bts {
+   using BCG;
+
+   using ProtoBuf;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   public class ProjectData {
+      // schema from the base stream
+      public readonly IBlueTableSchema in_schema;
+      // this is the projected schema
+      public readonly IBlueTableSchema out_schema;
+      // projected cols to the input cols, for cols who's names are changed
+      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
+      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
+                  IDictionary<string, string> mapped_to_orig_col_map_) {
+         in_schema = in_schema_;
+         out_schema = out_schema_;
+         mapped_to_orig_col_map = mapped_to_orig_col_map_;
+      }
+   }
+
+   public class NewProjectRow : IBlueRow {
+      #region IBlueRow members
+      private ProjectData _project_data;
+      private IBlueRow _row;
+      public NewProjectRow(ProjectData project_data, IBlueRow row) {
+         _project_data = project_data;
+         _row = row;
+      }
+      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
+      #endregion
+
+      #region ICollection members
+      public int Count { get { return schema.columnNames.Length; } }
+
+      public bool IsSynchronized { get { return false; } }
+
+      public object SyncRoot { get { return this; } }
+
+      public void CopyTo(Array array, int index) {
+         int i = index;
+         foreach (object o in this as IEnumerable)
+            array.SetValue(o, i++);
+      }
+      #endregion
+
+      #region IDictionary members
+      public bool IsFixedSize { get { return true; } }
+
+      public bool IsReadOnly { get { return true; } }
+
+      public object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            return this[(int)key];
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public object this[int index] {
+         get {
+            return this[schema.columnInfo[index].colName];
+         }
+      }
+
+      public object this[string col_name] {
+         get {
+            string mapped_to_col = null;
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+               return _row[mapped_to_col];
+            }
+            return _row[col_name];
+         }
+      }
+
+      public ICollection Keys { get { return schema.columnNames; } }
+
+      public ICollection Values {
+         get {
+            object[] retVal = new object[schema.columnNames.Length];
+            CopyTo(retVal, 0);
+            return retVal;
+         }
+      }
+
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public bool Contains(object key) { return schema[(string)key] != null; }
+
+      /// <summary>
+      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+      /// iterate through the values, typecast this into an IEnumerable
+      /// </summary>
+      /// <returns></returns>
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      #endregion
+
+   }
+
+   public class NewProjectStream : ViewStream, IStreamMapper {
+
+      private ViewStream _stream;
+      private string _project_expression;
+      private ProjectData _project_data = null;
+      //public ProjectData 
+      public NewProjectStream(ViewStream vs, string project_expression) {
+         _stream = vs;
+         _project_expression = project_expression;
+         _stream = new MapStream(vs, this);
+         mySetRealSource(_stream);
+      }
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+         _project_data = _create_project_data(input_schema, _project_expression);
+         return _project_data.out_schema;
+      }
+
+      public IBlueRow map(IBlueRow row) {
+         return new NewProjectRow(_project_data, row);
+      }
+
+      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
+                                                      string project_expression) {
+         project_expression = project_expression.Trim();
+         string[] project_toks = project_expression.Split(' ');
+         HashSet<string> removed_cols = new HashSet<string>();
+         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
+         bool hide_all = false;
+         foreach (string p_tok in project_toks) {
+            if (p_tok.StartsWith("!")) {
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
+            } else if (p_tok.Contains(":")) {
+               string[] map_toks = p_tok.Split(":");
+               string orig_col = map_toks[0];
+               string mapped_col = map_toks[1];
+               mapped_to_orig_map.Add(mapped_col, orig_col);
+               orig_to_mapped.Add(orig_col, mapped_col);
+            } else {
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
+            }
+         }
+
+         // make the new schema
+         List<ColumnInfo> new_cols = new List<ColumnInfo>();
+         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
+            if (removed_cols.Contains(ci.colName)) {
+               continue;
+            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
+               string mapped_name = orig_to_mapped[ci.colName];
+               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            } else {
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               //}
+            }
+         }
+         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs

@@ -1,20 +1,17 @@
 namespace BCG {
-   using mbts.spark;
-
    using Spark.Core.Collections;
 
-   using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
 
+   [DataStreamUserRespresentation("oldproject")]
    public class ProjectStream : ViewStream
    {
       private ProjectHelper myHelper;
       private string myColExp, myTableName;
       private ViewStream myVs;
       private IDictionary<string, string> myDic;
-      private HighPerformanceTimer _timer = new();
       /// <summary>
       /// </summary>
       /// <param name="vs">stream to susbsribe to as the input stream</param>
@@ -70,10 +67,8 @@ namespace BCG {
          DataPacket dp = newValue as DataPacket;
          if (myHelper != null)
          {
-            _timer.start();
             IBlueRowCollection dels, adds;
             BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
-            _timer.stop();
             if (dels.Count + adds.Count > 0)
                myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
                                             new ProjectedTable (adds, myHelper),

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs

@@ -0,0 +1,122 @@
+using System;
+using System.Data;
+using System.Data.SQLite;
+using System.Diagnostics;
+using System.Collections;
+
+
+using BCG;
+using System.Linq;
+using System.Data.Entity;
+using System.Collections.Generic;
+using System.Data.SQLite;
+
+
+namespace bts.SilverPine {
+
+   public static class SQLiteHelper {
+      public static string location = "C:\\Users\\m_tah\\development\\data\\";
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query) {
+         // Create a new DataTable to store the results
+         DataTable dataTable = new DataTable();
+
+         // Create and open a SQLite connection using the provided connection string
+         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
+            try {
+               connection.Open();
+
+               // Create a SQLite command with the provided query
+               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
+                  // Use SQLiteDataAdapter to fill the DataTable
+                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
+                     adapter.Fill(dataTable);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         // Return the populated DataTable
+         return dataTable;
+      }
+
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
+      }
+
+      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            try {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader()) {
+                  while (reader.Read()) {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++) {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
+      }
+
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
+      }
+   }
+}
+

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs

@@ -0,0 +1,73 @@
+namespace bts.SilverPine {
+   using System;
+   using System.Data;
+   using System.Data.SQLite;
+   using System.Diagnostics;
+   using System.Collections;
+
+
+   using BCG;
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+
+      private SQLiteStream(string connection_string, string query, bool noop) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+      public SQLiteStream(string db_name, string query)
+      :
+      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
+
+      public SQLiteStream(string query)
+      :
+      this(SQLiteHelper.db_name, query) {
+      }
+
+
+      private void _read_from_sqlite_and_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Restart();
+         #region perf measures
+         // SQLiteHelper.get_table(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.get_table_2(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         #endregion
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -3,12 +3,15 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
+using System.Diagnostics;
+using Microsoft.Data.Analysis;
+using Parquet;
+using Parquet.Data;
 
-namespace BCG
-{
+
+namespace BCG {
    [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream
-   {
+   public class CSVSaverStream : ViewStream, ISideEffectStream {
       private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("fileName", 1, typeof(string)),
          new ColumnInfo("nWrites", 2, typeof(int)),
@@ -20,11 +23,9 @@ namespace BCG
       private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
 
       public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
-      { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
 
-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
-      {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
          myStream = vs;
          myFullyQualifiedFileName = fullyQualifiedFileName;
          myOverwrite = overwrite;
@@ -36,18 +37,14 @@ namespace BCG
 
       protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
 
-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
          IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
-         {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
             myNWrites++;
             int i = 0;
             StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState)
-            {
-               if (i++ == 0)
-               {
+            foreach (IBlueRow r in totalState) {
+               if (i++ == 0) {
                   string header = String.Empty;
                   foreach (string col in r.schema.columnNames)
                      if (col != "id")
@@ -59,20 +56,17 @@ namespace BCG
                writer.Flush();
             }
             writer.Close();
-            if (myFireResultSnapshot)
-            {
+            if (myFireResultSnapshot) {
                IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
                BlueTable nRows = new BlueTable(myResultSchema);
                nRows.addRow(nr);
                myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            }
-            else
+            } else
                myFireEvent(newValue);
          }
       }
 
-      private string myGetNewFileName()
-      {
+      private string myGetNewFileName() {
          string[] tags = myFullyQualifiedFileName.Split('.');
          if (myOverwrite)
             return tags[0] + ".csv";
@@ -80,14 +74,11 @@ namespace BCG
             return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
       }
 
-      private static string myGetCSVRow(IBlueRow r)
-      {
+      private static string myGetCSVRow(IBlueRow r) {
          string rv = String.Empty;
          int i = 0;
-         foreach (string col in r.schema.columnNames)
-         {
-            if (col != "id")
-            {
+         foreach (string col in r.schema.columnNames) {
+            if (col != "id") {
                string colText = String.Empty;
                if (r[col] != null && r[col] != DBNull.Value)
                   colText = r[col].ToString();
@@ -99,8 +90,7 @@ namespace BCG
          return rv;
       }
 
-      private string myFormateDateTime(DateTime dt)
-      {
+      private string myFormateDateTime(DateTime dt) {
          return
          string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                         dt.Year.ToString("0000"),
@@ -114,52 +104,49 @@ namespace BCG
    }
 
    [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream
-   {
-      private IBlueRowCollection myRows;
-
-      public CSVReaderStream(string fileAddress)
-      {
-         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
+   public class CSVReaderStream : ViewStream {
+      private readonly string _file_address;
+      Stopwatch _stopwatch = new Stopwatch();
+      public CSVReaderStream(string fileAddress) {
+         _file_address = fileAddress;
       }
 
-      protected override void myOnFirstSubscriber()
-      {
-         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
+      protected override void myOnFirstSubscriber() {
+         _stopwatch.Start();
+         DataFrame df = DataFrame.LoadCsv(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
       }
    }
 
-   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream
-   {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
-      {
+   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
          mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
       }
    }
 
-   public class CSVReaderStreamMapper : IStreamMapper
-   {
+   public class CSVReaderStreamMapper : IStreamMapper {
       private IBlueTableSchema mySchema;
       private readonly string[] myOutputSchemaStringArray;
 
-      public CSVReaderStreamMapper(string outputSchema)
-      {
+      public CSVReaderStreamMapper(string outputSchema) {
          myOutputSchemaStringArray = outputSchema.Split(' ');
       }
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
             ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray)
-            {
+            foreach (string col in myOutputSchemaStringArray) {
                string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++)
-               {
-                  if (ci[i].colName.ToLower().Equals(colName))
-                  {
+               for (int i = 0; i < ci.Length; i++) {
+                  if (ci[i].colName.ToLower().Equals(colName)) {
                      Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                      ci[i] = new ColumnInfo(ci[i].colName, i, t);
                      break;
@@ -171,74 +158,52 @@ namespace BCG
          return mySchema;
       }
 
-      public IBlueRow map(IBlueRow row)
-      {
+      public IBlueRow map(IBlueRow row) {
          object[] vals = new object[row.schema.columnInfo.Length];
          row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++)
-         {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
             Type t = mySchema.columnInfo[i].colType;
-            try
-            {
+            try {
                vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            }
-            catch
-            {
-               if (t.Equals(typeof(Int16)))
-               {
+            } catch {
+               if (t.Equals(typeof(Int16))) {
                   Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result))
-                  {
+                  if (Int16.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int16)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
-               }
-               else if (t.Equals(typeof(Int32)))
-               {
+               } else if (t.Equals(typeof(Int32))) {
                   Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result))
-                  {
+                  if (Int32.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int32)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
-               }
-               else if (t.Equals(typeof(Int64)))
-               {
+               } else if (t.Equals(typeof(Int64))) {
                   Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result))
-                  {
+                  if (Int64.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int64)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
                }
                vals[i] = DBNull.Value;
@@ -247,8 +212,7 @@ namespace BCG
          return new BlueRow(mySchema, vals);
       }
 
-      private Type myGetType(string type)
-      {
+      private Type myGetType(string type) {
          type = type.ToLower();
          if (type.Contains("int"))
             return typeof(int);

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -24,7 +24,6 @@ namespace bts {
             var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
             myFireEvent(dp_out);
 
-
         }
 
 

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -69,18 +69,14 @@ public class SortRow : IComparable, IBlueRow {
    /// The key can either be the string corrsponding to the column name or
    /// the order of the column
    /// </summary>
-   public object this[object key] {
-      get {
-         if (key is string)
-            return this[(string)key];
-         return this[(int)key];
-      }
-      set { throw new NotSupportedException("The IBlueRow is readonly"); }
+   public object? this[object key] {
+      get => key is string str ? this[str] : this[(int)key];
+      set => throw new NotSupportedException("The IBlueRow is readonly");
    }
 
-   public object this[int index] => row[index];
+   public object? this[int index] => row[index];
 
-   public object this[string colName] => row[colName];
+   public object? this[string colName] => row[colName];
 
    public ICollection Keys => row.Keys;