refactor: Replace if/else with switch cases

cleanup: Remove over-engineered subcommands
We don't need explicit subcommands to 'run onixs' md (just use config).
2025-07-24 14:07:58 +00:00 · 2025-07-24 14:07:58 +00:00 · 2025-07-24 14:07:58 +00:00 · 2025-07-24 14:07:58 +00:00 · 2025-07-24 14:07:58 +00:00 · 2025-07-24 14:07:58 +00:00
1915 changed files with 84551 additions and 30409 deletions
@@ -9,7 +9,7 @@
      ]
    },
    "fable": {
-      "version": "4.0.6",
+      "version": "4.24.0",
      "commands": [
        "fable"
      ]
@@ -25,6 +25,12 @@
      "commands": [
        "fake"
      ]
+    },
+    "dotnet-t4": {
+      "version": "3.0.0",
+      "commands": [
+        "t4"
+      ]
    }
  }
-}
+}
@@ -65,7 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
-RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8

 # Node
 ARG node_major=18
@@ -129,5 +129,3 @@ RUN \
   -G sudo \
   --password "$(openssl passwd -1 "$docker_pass")" \
   "$docker_user"
-
-RUN usermod -aG docker "$docker_user"
@@ -1,15 +1,10 @@
 {
  "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
  "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
+    "memory": "8gb",
+    "storage": "10gb"
  },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
  "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
  "updateRemoteUserUID": true,
  "shutdownAction": "none",
  // Note: If running multiple containers on the same host, each must use a
@@ -17,26 +12,19 @@
  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
  "runArgs": [
    "--shm-size",
-    "2gb",
-    "--network",
-    "host"
+    "8gb"
  ],
  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
  ],
  "customizations": {
    "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
      "extensions": [
        "eamodio.gitlens",
        "llvm-vs-code-extensions.vscode-clangd",
        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
      ]
    }
  }
@@ -0,0 +1,49 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+SPARKD__PYTHON_REQUIREMENTS=requirements.txt
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -1,4 +1,3 @@
 *.cmd text=auto eol=crlf
 *.cs text=auto eol=crlf
 *.sh text=auto eol=lf
-*.py text=auto eol=lf
@@ -11,6 +11,7 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
+!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -59,3 +60,5 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local
+.secrets
@@ -1,6 +1,14 @@
 {
  "version": "0.2.0",
  "configurations": [
+    {
+      "name": "Python Debugger: Current File with Arguments",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "args": "${command:pickArgs}"
+    },
    {
      "name": ".NET Core Attach",
      "type": "coreclr",
@@ -46,8 +54,8 @@
      "type": "coreclr",
      "request": "launch",
      "preLaunchTask": "Build DataStream WebApp (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
      "env": {
        "ASPNETCORE_HTTP_PORT": "8080",
        "ASPNETCORE_HTTPS_PORT": "8443",
@@ -59,8 +67,8 @@
      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
      "type": "coreclr",
      "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
      "env": {
        "ASPNETCORE_HTTP_PORT": "8080",
        "ASPNETCORE_HTTPS_PORT": "8443",
@@ -80,10 +88,10 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
+      "cwd": "build/Debug/bin/net8.0",
      "env": {
        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -98,13 +106,57 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
      "env": {
        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "run"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests2",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader2]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
@@ -117,7 +169,7 @@
          "--config",
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
        "env": {
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
@@ -137,7 +189,7 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
      "env": {
        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
      },
@@ -170,11 +222,19 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "args": [
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -191,7 +251,40 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -200,7 +293,6 @@
      "name": "Sparkd",
      "type": "cppdbg",
      "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
@@ -298,6 +390,16 @@
      "description": "Config Server URL",
      "default": "http://localhost:8000/config",
    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
    {
      "id": "isSparkApiEnabled",
      "type": "pickString",
@@ -0,0 +1 @@
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
@@ -490,7 +490,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -510,7 +510,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -530,7 +530,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -554,7 +554,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -577,7 +577,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -600,7 +600,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -706,7 +706,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -729,7 +729,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -751,7 +751,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -774,7 +774,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -796,7 +796,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -811,7 +811,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -827,7 +827,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -843,7 +843,22 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -868,7 +883,7 @@
        "${input:sparkConfigQuery}",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -894,6 +909,24 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Run Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
    {
      "label": "Run Sparkd Config Query",
      "dependsOn": "Build Sparkd",
@@ -915,6 +948,65 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Test Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+      "command": "",
+      "windows": {
+        "command": "Spark.Console.exe",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        }
+      },
+    },
+    {
+      "label": "Test Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
+    {
+      "label": "Test Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
    {
      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
      "type": "process",
@@ -1033,6 +1125,16 @@
      "id": "sparkdConfigQuery",
      "type": "promptString",
      "description": "Config Type(s)"
-    }
+    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
  ]
 }
@@ -1,7 +1,10 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)

+get_version_from_git()
 if(WIN32)
  set(platform "windows")
 else()
@@ -82,9 +85,15 @@ else()
    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
+    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
+
+    # ** Binance Coin Futures **
+    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

    # ** Deribit **
    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -103,15 +112,15 @@ else()
  endif()

  # subdirs
+  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_system)
-    add_subdirectory(src/lib_http)
-    add_subdirectory(src/lib_domain)
-    add_subdirectory(src/lib_python_md)
+    add_subdirectory(src/lib_python_interop)
+    add_subdirectory(src/lib_onixs_fix)
+    add_subdirectory(src/lib_binance)
+    add_subdirectory(src/lib_okx_execution)
    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib_spark)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -24,12 +24,13 @@
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -49,11 +50,20 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "DATABENTO_USE_EXTERNAL_JSON": {
+          "type": "BOOL",
+          "value": "ON"
+        },
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -164,12 +174,13 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_EXPORT_COMPILE_COMMANDS": {
          "type": "BOOL",
          "value": "ON"
        },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -207,8 +218,8 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
      }
    },

@@ -240,9 +251,8 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "CONFIG_PROFILES": "default;data_stream",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -295,14 +305,6 @@
      "inherits": "sparkd"
    },

-    {
-      "name": "sparkd-debug-clang-j6",
-      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang | jobs=6)",
-      "inherits": "sparkd",
-      "jobs": 6
-    },
-
    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
@@ -14,6 +14,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +98,102 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -80,4 +204,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal
@@ -0,0 +1,48 @@
+include_guard(GLOBAL)
+
+function(get_version_from_git)
+  find_package(Git QUIET)
+  if(NOT Git_FOUND)
+    message(WARNING "Git not found")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --tags --always
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_TAG
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE GIT_RESULT
+  )
+
+  if(NOT GIT_RESULT EQUAL 0)
+    message(WARNING "Failed to get git tag")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
+
+  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
+  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
+
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
+  else()
+    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
+  endif()
+endfunction()
@@ -72,6 +72,14 @@ function(init_library)
  else()
    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
  target_include_directories(
    ${_name}
    PUBLIC
@@ -90,6 +98,7 @@ function(init_library)
    ${_name}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
  )
  # Install main lib
@@ -103,7 +112,7 @@ function(init_library)
  )

  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})

    get_target_property(_lib_type ${_name} TYPE)
@@ -0,0 +1,71 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(init_python_module)
+  set(_options)
+  set(_one_value_args
+    NAME
+  )
+  set(_multi_value_args
+    SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
+
+  set(module_proj ${_name}_module)
+  set(${module_proj}_SOURCES ${_sources})
+  pybind11_add_module(${module_proj} SHARED ${_sources})
+  target_link_libraries(
+    ${module_proj}
+    PUBLIC
+      pybind11::embed
+      spark
+      ${_name}
+  )
+  target_include_directories(
+    ${module_proj}
+    PRIVATE
+      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
+  )
+  target_include_directories(
+    ${module_proj}
+    PUBLIC
+      ${Python3_INCLUDE_DIRS}
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+    DEPENDS ${_name} ${module_proj}
+    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMENT "Generating py stubs"
+  )
+  add_custom_target(
+    "${_name}_py_stubs"
+    ALL
+    DEPENDS
+      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+  )
+  install(
+    TARGETS ${module_proj}
+    EXPORT ${module_proj}Targets
+    RUNTIME
+      DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+  foreach(dir ${pybind11_INCLUDE_DIRS})
+    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  endforeach()
+  install(
+    EXPORT ${module_proj}Targets
+    FILE ${module_proj}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+endfunction()
@@ -0,0 +1,65 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+
+function(init_python_plugin)
+  set(_options)
+  set(_one_value_args NAME PLUGIN_NAME)
+  set(_multi_value_args SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  if (NOT DEFINED _ARGS_PLUGIN_NAME)
+    set(_plugin_name ${_name}_plugins)
+  endif()
+  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
+  if (NOT DEFINED _ARGS_SOURCES)
+    file(
+      GLOB_RECURSE ${_plugin_name}_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${_plugin_name}/*.py)
+  endif()
+  set(src_outs)
+  foreach(_src ${${_plugin_name}_SOURCES})
+    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
+    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
+    add_custom_command(
+      OUTPUT ${_src_out}
+      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
+      DEPENDS ${_src_in}
+      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
+    )
+    list(APPEND src_outs ${_src_out})
+  endforeach()
+  
+  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
+  
+  add_library(${_plugin_name} INTERFACE)
+  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+      INTERFACE_PYTHON_SCRIPTS_DIR
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
+  )
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
+  )
+  
+  install(
+    TARGETS ${_plugin_name}
+    EXPORT ${_plugin_name}Targets
+    DESTINATION ${CMAKE_INSTALL_BINDIR}
+  )
+  
+  install(
+    EXPORT ${_plugin_name}Targets
+    FILE ${_plugin_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
+endfunction()
@@ -0,0 +1,182 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE}
+          ARGS
+            --cpp_out=${_ARGS_CXX_OUT_DIR}
+            --plugin=protoc-gen-service=${_plugin_cmd}
+            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
+            -I ${PROTOBUF_INCLUDE_DIRS}
+            -I ${_protobuf_src_dir}
+            ${ABS_FIL}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_cmd}
+          ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM
+      )
+    else()
+      add_custom_command(
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM
+      )
+    endif()
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
@@ -15,7 +15,11 @@ function(protobuf_generate_cpp)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})

  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
  endif()
  set(${_ARGS_CXX_SOURCES})
  set(${_ARGS_CXX_HEADERS})
@@ -35,6 +39,8 @@ function(protobuf_generate_cpp)
    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
    add_custom_command(
      OUTPUT
        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
@@ -42,7 +48,7 @@ function(protobuf_generate_cpp)
      COMMAND
        ${_ARGS_PROTOC}
      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
      DEPENDS ${_abs_fil}
      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
      VERBATIM
@@ -74,3 +80,4 @@ function(protobuf_generate_cpp)
  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
 endfunction()
+
@@ -1,3 +1,5 @@
+include_guard(GLOBAL)
+
 function(protobuf_generate_cpp_config)
  set(_options)
  set(_one_value_args
@@ -33,7 +35,7 @@ function(protobuf_generate_cpp_config)

  # Generate static defaults. Ideally, this would be a single string, but we use an array because
  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "const char * config_static_defaults[] = { ")
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")

  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
    file(READ ${_cfg_file_path} _cfg_file_content)
@@ -56,3 +58,4 @@ function(protobuf_generate_cpp_config)
    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
  endif()
 endfunction()
+
@@ -20,16 +20,14 @@ endfunction()

 function(sym_link_file)
  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
  )
  if (DEFINED _ARGS_INSTALL_DEST)
    install(
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+function(sym_link_python_module)
+  set(_options)
+  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
+                        "${_multi_value_args}" ${ARGN})
+  if(NOT DEFINED _ARGS_LIB_NAME)
+    message(
+      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
+  endif()
+  set(_lib_name ${_ARGS_LIB_NAME})
+  set(_module_name ${_ARGS_MODULE_NAME})
+  if(NOT DEFINED _ARGS_MODULE_NAME)
+    set(_module_name ${_lib_name}_module)
+  endif()
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  set(_deps ${_lib_name} ${_module_name})
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
+  endif()
+  add_dependencies(${PROJECT_NAME} ${_deps})
+  sym_link_local_lib(${_lib_name})
+  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
+  sym_link_file(
+    SOURCE
+    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
+    DEST
+    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
+    INSTALL_DEST
+    ${CMAKE_INSTALL_BINDIR}/)
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
+    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
+                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
+  endif()
+endfunction()
@@ -0,0 +1,11 @@
+#ifndef BTS_SPARK_VERSION_H
+#define BTS_SPARK_VERSION_H
+
+#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
+#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
+#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
+#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
+#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
+#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
+
+#endif
@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -1,51 +0,0 @@
-services:
-  sparkd:
-    build:
-      dockerfile: Dockerfile
-      context: ./sparkd
-    command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
-
-  pubsub:
-    build:
-      dockerfile: pubsub.Dockerfile
-      context: ./DataStream
-    command: [ "./PubSubServer" ]
-    environment:
-      - DOTNET_ENVIRONMENT=Production
-    ports:
-      - 22019:22019
-
-  md-publisher:
-    build:
-      dockerfile: md-publisher.Dockerfile
-      context: ./DataStream
-    depends_on:
-      - pubsub
-      - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  webapp:
-    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: webapp.Dockerfile
-      context: ./DataStream
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_HTTP_PORT=8080
-      - ASPNETCORE_HTTPS_PORT=8443
-      - ASPNETCORE_ENVIRONMENT=Staging
@@ -1,62 +1,100 @@
 services:
  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/server/sparkd/Dockerfile
-    command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
-
-  pubsub:
-    environment:
-      - DOTNET_ENVIRONMENT=Staging
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
    ports:
-      - 22019:22019
-    command: [ "./PubSubServer" ]
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"

-  md-publisher:
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-    depends_on:
-      - pubsub
-      - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  webapp:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"

-  api-server:
-    profiles:
-      - api-server
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]

  sparkd-ping-test:
    depends_on:
@@ -70,3 +108,58 @@ services:
        USER "root"
        RUN apt-get update && apt-get -y install iputils-ping
    command: [ "ping", "sparkd" ]
+
+  pubsub:
+    environment:
+      - DOTNET_ENVIRONMENT=Staging
+    build:
+      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+    ports:
+      - 22019:22019
+    command: [ "./PubSubServer" ]
+
+  md-publisher:
+    build:
+      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+    depends_on:
+      - pubsub
+      - sparkd
+    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
+    environment:
+      - DOTNET_ENVIRONMENT=MarketData
+
+  webapp:
+    ports:
+      - 8080:8080
+      - 8443:8443
+    build:
+      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
+    command: [ "./Bts.DataStreamWebApp.Server" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
+  api-server:
+    depends_on:
+      - sparkd
+    ports:
+      - 8008:8008
+      - 9443:9443
+    build:
+      dockerfile: src/client/spark_api_server/Dockerfile
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
+virtual void on_status(SecurityKey, co2::TradingStatus) override;
 ```

 # Child Orders
@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts
@@ -1,22 +1,42 @@
 astroid==2.7.3
 autopep8==1.5.7
+blinker==1.9.0
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
+click==8.1.8
 colorama==0.4.4
+Flask==3.1.0
 idna==3.4
 isort==5.9.3
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
+MarkupSafe==3.0.2
 mccabe==0.6.1
+numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
+pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
+pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+websocket-client==1.8.0
+Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -18,6 +18,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\clie
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -90,6 +98,38 @@ Global
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.3" />
  </ItemGroup>

  <ItemGroup>
@@ -24,8 +24,6 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -60,12 +58,10 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

-#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
-#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -353,17 +349,15 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
+            NextBlk = BlockReaderFactory.Get(stream);
        }

-#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
-#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -434,43 +428,19 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }

-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
+            else if (fallbackCodePage >= 0) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
+                encoding = GuessEncoding(stream);
+
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -488,6 +458,14 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
+
+            int b3 = stream.ReadByte();
+            // Check for UTF-32 BOM (Byte Order Mark)
+            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
+                return 12001; // UTF-32BE
+            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
+                return 12000; // UTF-32LE
+
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -495,7 +473,150 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-#endif // !BYTEMODE
+
+        private static Encoding GuessEncoding(Stream stream)
+        {
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf16(stream, out Encoding encoding))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return encoding;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf8(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.UTF8;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsAscii(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.ASCII;
+            }
+
+#if !NET462_OR_GREATER
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsLatin1(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.Latin1;
+            }
+#endif
+            stream.Seek(0, SeekOrigin.Begin);
+            return Encoding.Default;
+        }
+
+        private static bool IsUtf8(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                if ((b & 0x80) == 0) // 0xxxxxxx
+                {
+                    continue;
+                }
+                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsValidUtf8ContinuationByte(Stream stream)
+        {
+            int b = stream.ReadByte();
+            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
+        }
+
+        private static bool IsUtf16(Stream stream, out Encoding encoding)
+        {
+            encoding = Encoding.Default;
+
+            if (stream.Length >= 2)
+            {
+                // Read the first two bytes
+                int b0 = stream.ReadByte();
+                int b1 = stream.ReadByte();
+
+                // Check for UTF-16 BOM (Byte Order Mark)
+                if (b0 == 0xFF && b1 == 0xFE)
+                {
+                    encoding = Encoding.Unicode;
+                    return true; // UTF-16LE
+                }
+                if (b0 == 0xFE && b1 == 0xFF)
+                {
+                    encoding = Encoding.BigEndianUnicode;
+                    return true; // UTF-16BE
+                }
+
+                // Reset the stream position
+                stream.Seek(0, SeekOrigin.Begin);
+
+                // Check for common UTF-16 patterns
+                byte[] buffer = new byte[2];
+                while (stream.Read(buffer, 0, 2) == 2)
+                {
+                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
+                    {
+                        encoding = Encoding.BigEndianUnicode;
+                        return true; // Likely UTF-16BE
+                    }
+                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
+                    {
+                        encoding = Encoding.Unicode;
+                        return true; // Likely UTF-16LE
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        private static bool IsAscii(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0x7F)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsLatin1(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0xFF)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -540,180 +661,6 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -12,18 +12,6 @@ module.exports = {
  settings: { react: { version: '18.3.1' } },
  plugins: ['react-refresh'],
  rules: {
-    "import/order": [
-      "error",
-      {
-        "groups": [
-          "builtin",
-          "external",
-          "internal",
-          ["parent", "sibling", "index"]
-        ],
-        "newlines-between": "always"
-      }
-    ],
    'react-refresh/only-export-components': [
      'warn',
      { allowConstantExport: true },
@@ -32,7 +32,6 @@
        "@types/react-dom": "^18.3.0",
        "@vitejs/plugin-react": "^4.3.1",
        "eslint": "^8.57.0",
-        "eslint-plugin-import": "^2.29.1",
        "eslint-plugin-react": "^7.34.2",
        "eslint-plugin-react-hooks": "^4.6.2",
        "eslint-plugin-react-refresh": "^0.4.7",
@@ -3179,12 +3178,6 @@
      "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==",
      "dev": true
    },
-    "node_modules/@types/json5": {
-      "version": "0.0.29",
-      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
-      "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==",
-      "dev": true
-    },
    "node_modules/@types/parse-json": {
      "version": "4.0.2",
      "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
@@ -3499,26 +3492,6 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
-    "node_modules/array.prototype.findlastindex": {
-      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.5.tgz",
-      "integrity": "sha512-zfETvRFA8o7EiNn++N5f/kaCw221hrpGsDmcpndVupkPzEc1Wuf3VgC0qby1BbHs7f5DVYjgtEU2LLh5bqeGfQ==",
-      "dev": true,
-      "dependencies": {
-        "call-bind": "^1.0.7",
-        "define-properties": "^1.2.1",
-        "es-abstract": "^1.23.2",
-        "es-errors": "^1.3.0",
-        "es-object-atoms": "^1.0.0",
-        "es-shim-unscopables": "^1.0.2"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/array.prototype.flat": {
      "version": "1.3.2",
      "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz",
@@ -4333,121 +4306,6 @@
        "url": "https://opencollective.com/eslint"
      }
    },
-    "node_modules/eslint-import-resolver-node": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz",
-      "integrity": "sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==",
-      "dev": true,
-      "dependencies": {
-        "debug": "^3.2.7",
-        "is-core-module": "^2.13.0",
-        "resolve": "^1.22.4"
-      }
-    },
-    "node_modules/eslint-import-resolver-node/node_modules/debug": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
-      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
-      "dev": true,
-      "dependencies": {
-        "ms": "^2.1.1"
-      }
-    },
-    "node_modules/eslint-import-resolver-node/node_modules/resolve": {
-      "version": "1.22.8",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
-      "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==",
-      "dev": true,
-      "dependencies": {
-        "is-core-module": "^2.13.0",
-        "path-parse": "^1.0.7",
-        "supports-preserve-symlinks-flag": "^1.0.0"
-      },
-      "bin": {
-        "resolve": "bin/resolve"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/eslint-module-utils": {
-      "version": "2.8.1",
-      "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.1.tgz",
-      "integrity": "sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==",
-      "dev": true,
-      "dependencies": {
-        "debug": "^3.2.7"
-      },
-      "engines": {
-        "node": ">=4"
-      },
-      "peerDependenciesMeta": {
-        "eslint": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/eslint-module-utils/node_modules/debug": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
-      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
-      "dev": true,
-      "dependencies": {
-        "ms": "^2.1.1"
-      }
-    },
-    "node_modules/eslint-plugin-import": {
-      "version": "2.29.1",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz",
-      "integrity": "sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==",
-      "dev": true,
-      "dependencies": {
-        "array-includes": "^3.1.7",
-        "array.prototype.findlastindex": "^1.2.3",
-        "array.prototype.flat": "^1.3.2",
-        "array.prototype.flatmap": "^1.3.2",
-        "debug": "^3.2.7",
-        "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.9",
-        "eslint-module-utils": "^2.8.0",
-        "hasown": "^2.0.0",
-        "is-core-module": "^2.13.1",
-        "is-glob": "^4.0.3",
-        "minimatch": "^3.1.2",
-        "object.fromentries": "^2.0.7",
-        "object.groupby": "^1.0.1",
-        "object.values": "^1.1.7",
-        "semver": "^6.3.1",
-        "tsconfig-paths": "^3.15.0"
-      },
-      "engines": {
-        "node": ">=4"
-      },
-      "peerDependencies": {
-        "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8"
-      }
-    },
-    "node_modules/eslint-plugin-import/node_modules/debug": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
-      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
-      "dev": true,
-      "dependencies": {
-        "ms": "^2.1.1"
-      }
-    },
-    "node_modules/eslint-plugin-import/node_modules/doctrine": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
-      "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==",
-      "dev": true,
-      "dependencies": {
-        "esutils": "^2.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
    "node_modules/eslint-plugin-react": {
      "version": "7.34.2",
      "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.34.2.tgz",
@@ -5745,15 +5603,6 @@
        "node": "*"
      }
    },
-    "node_modules/minimist": {
-      "version": "1.2.8",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
-      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
-      "dev": true,
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/ms": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -5882,20 +5731,6 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
-    "node_modules/object.groupby": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/object.groupby/-/object.groupby-1.0.3.tgz",
-      "integrity": "sha512-+Lhy3TQTuzXI5hevh8sBGqbmurHbbIjAi0Z4S63nthVLmLxfbj4T54a4CfZrXIrt9iP4mVAPYMo/v99taj3wjQ==",
-      "dev": true,
-      "dependencies": {
-        "call-bind": "^1.0.7",
-        "define-properties": "^1.2.1",
-        "es-abstract": "^1.23.2"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      }
-    },
    "node_modules/object.hasown": {
      "version": "1.1.4",
      "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.4.tgz",
@@ -6749,15 +6584,6 @@
        "node": ">=8"
      }
    },
-    "node_modules/strip-bom": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-      "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==",
-      "dev": true,
-      "engines": {
-        "node": ">=4"
-      }
-    },
    "node_modules/strip-json-comments": {
      "version": "3.1.1",
      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
@@ -6851,30 +6677,6 @@
        "node": ">=4"
      }
    },
-    "node_modules/tsconfig-paths": {
-      "version": "3.15.0",
-      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz",
-      "integrity": "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==",
-      "dev": true,
-      "dependencies": {
-        "@types/json5": "^0.0.29",
-        "json5": "^1.0.2",
-        "minimist": "^1.2.6",
-        "strip-bom": "^3.0.0"
-      }
-    },
-    "node_modules/tsconfig-paths/node_modules/json5": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
-      "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
-      "dev": true,
-      "dependencies": {
-        "minimist": "^1.2.0"
-      },
-      "bin": {
-        "json5": "lib/cli.js"
-      }
-    },
    "node_modules/tslib": {
      "version": "2.6.3",
      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
@@ -34,7 +34,6 @@
    "@types/react-dom": "^18.3.0",
    "@vitejs/plugin-react": "^4.3.1",
    "eslint": "^8.57.0",
-    "eslint-plugin-import": "^2.29.1",
    "eslint-plugin-react": "^7.34.2",
    "eslint-plugin-react-hooks": "^4.6.2",
    "eslint-plugin-react-refresh": "^0.4.7",
@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig, loadEnv } from 'vite';
+import { defineConfig } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -29,12 +29,13 @@ if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
  }
 }

+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:8443';
+
 export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
-  const target = env.ASPNETCORE_HTTPS_PORT
-    ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
-    : env.ASPNETCORE_URLS
-      ? env.ASPNETCORE_URLS.split(';')[0]
-      : 'https://localhost:8443';
  return {
    plugins: [plugin()],
    resolve: {
@@ -1,14 +1,4 @@
 {
-  "Kestrel": {
-    "Endpoints": {
-      "Http": {
-        "Url": "http://localhost:8080"
-      },
-      "Https": {
-        "Url": "https://localhost:8443"
-      }
-    }
-  },
  "PubSub": {
    "Routing": {
      "PublisherRoutes": [
@@ -30,5 +30,15 @@
        "benzinga_news"
      ]
    }
-  ]
+  ],
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  }
 }
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -0,0 +1,11 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   [DataStreamUserRespresentation("AddColumnsStream")]
+   public class AddColumnsStream: ViewStream {
+      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
+         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,49 @@
+using System;
+
+namespace BCG {
+   [DataStreamUserRespresentation("CountStream")]
+   public class CountStream : ViewStream {
+      private DataPacket myLastFiredDataPacket;
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
+      private int myId = 0;
+      private ViewStream myVS;
+
+      public CountStream(ViewStream vs) {
+         myVS = vs;
+      }
+
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
+
+      protected override void myOnFirstSubscriber() {
+         myVS.onChange += myHandleChanges;
+      }
+      protected override void myOnLastSubscriber() {
+         myVS.onChange -= myHandleChanges;
+      }
+
+
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
+            myLastFiredDataPacket =
+               myLastFiredDataPacket == null ?
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
+                  :
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  myLastFiredDataPacket.newRows)
+            );
+      }
+
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
+         retVal.addRow
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+         return retVal;
+      }
+
+
+   }
+}
@@ -0,0 +1,38 @@
+namespace bts.BlueOrb {
+   using bts;
+
+   using BCG;
+
+   using System;
+   [DataStreamUserRespresentation("DateTimeExpanderStream")]
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_minute", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,157 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using System;
+   using System.Text.RegularExpressions;
+
+   [DataStreamUserRespresentation("FilterStream")]
+   public class FilterStream : ViewStream {
+      public class RegExpFilter : IStreamFilter {
+         private Regex[] myRegExps;
+         private string[] myCols;
+
+         public RegExpFilter(string col, string regExp)
+            : this(new string[] { col }, new string[] { regExp }) { }
+
+         public RegExpFilter(string[] cols, string[] regExps) {
+            myCols = cols;
+            myRegExps = new Regex[regExps.Length];
+            for (int i = 0; i < regExps.Length; i++)
+               myRegExps[i] = new Regex(regExps[i]);
+         }
+
+         #region IStreamFilter Members
+         public bool filter(IBlueRow dr) {
+            bool rv = true;
+            for (int i = 0; i < myCols.Length; i++) {
+               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+               if (!rv)
+                  return rv;
+            }
+            return rv;
+         }
+         #endregion
+      }
+      private BlueTable myLastFiredRows;
+      private IBlueRowCollection myLastReceivedRows;
+      private IStreamFilter myFilter;
+      private ViewStream myParentStream;
+      private string myFilterString;
+      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
+
+      #region constructors
+      public delegate bool OnFilter(IBlueRow r);
+      private class OnFilterWrapper : IStreamFilter {
+         public OnFilterWrapper(OnFilter f) { this.f = f; }
+
+         public bool filter(IBlueRow dr) { return f(dr); }
+
+         private OnFilter f;
+      }
+      public FilterStream(ViewStream vs, OnFilter f)
+         : this(vs, new OnFilterWrapper(f)) { }
+
+
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
+         {
+            myParentStream = vs;
+            myFilter = filter;
+            myFilterString = filter.ToString();
+            if (filter is IDynamicStreamFilter)
+               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         }
+      }
+
+      public FilterStream(ViewStream vs, string filterExpression)
+         : this(vs, new BooleanExpressionFilter(filterExpression)) {
+         myFilterString = filterExpression;
+      }
+
+      public FilterStream(ViewStream vs, string col, string regExp)
+         : this(vs, new RegExpFilter(col, regExp)) {
+         myFilterString = col + " , " + regExp;
+      }
+
+      public FilterStream(ViewStream vs)
+         : this(vs, new BinaryFilter(true)) {
+         myFilterString = true.ToString();
+      }
+      #endregion
+
+      #region overridden methods
+      protected override void myOnFirstSubscriber() =>
+         myParentStream.onChange += myHandleCallbacks;
+
+      protected override void myOnLastSubscriber() {
+         myLastFiredRows = null;
+         myParentStream.onChange -= myHandleCallbacks;
+      }
+
+      protected override void myFireEvent(object o) {
+         DataPacket dp = o as DataPacket;
+         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
+            base.myFireEvent(o);
+      }
+      #endregion
+
+      private void myHandleCallbacks(object newValue) {
+         DataPacket retVal;
+         DataPacket arrivedPacket = newValue as DataPacket;
+         myLastReceivedRows = arrivedPacket.newRows;
+         if (myLastFiredRows != null) {
+            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+               new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.deletedRows)
+               if (myLastFiredRows.containsID((int)r["id"])) {
+                  myLastFiredRows.deleteRow(r);
+                  (retVal.deletedRows as BlueTable).addRow(r);
+               }
+            foreach (IBlueRow r in arrivedPacket.addedRows)
+               if (myFilter.filter(r)) {
+                  myLastFiredRows.addRow(r);
+                  (retVal.addedRows as BlueTable).addRow(r);
+               }
+         } else {
+            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.newRows)
+               if (myFilter.filter(r))
+                  myLastFiredRows.addRow(r);
+         }
+         myFireEvent(retVal);
+      }
+
+      public IStreamFilter filter {
+         set {
+            {
+               myFilter = value;
+               if (numberOfSubscribers > 0) {
+                  if (myLastReceivedRows != null) {
+                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
+                     foreach (IBlueRow r in myLastReceivedRows)
+                        if (myFilter.filter(r))
+                           nRows.addRow(r);
+                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
+                        aRows = new BlueTable(nRows.schema);
+                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
+                     myLastFiredRows = nRows;
+                     myFireEvent(new DataPacket(nRows, aRows, dRows));
+                  }
+               }
+            }
+         }
+         get {            
+               return myFilter;
+         }
+      }
+
+      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
+
+      public override string ToString() {
+         return GetType().ToString() + " " + myFilterString;
+      }
+
+      private void myHandleFilterChange() {
+         filter = myFilter;
+      }
+   }
+}
@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;
+
+using bts;
+//using bts.SilverPine;
+using BCG;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("GroupByStream")]
+   public class GroupByStream : ViewStream {
+
+      public interface INonIncGroupByEvaluator {
+         // grouped_rows
+         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[] get_group_by_cols();
+      }
+
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
+      private ViewStream _main_stream;
+      private INonIncGroupByEvaluator _groupby_func;
+      private BlueRowHasher _row_hasher;
+      private string[] _groupby_cols;
+      private Stopwatch _timer = new();
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+
+      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+         _main_stream = main_stream;
+         _groupby_func = groupby_func;
+         _groupby_cols = _groupby_func.get_group_by_cols();
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+      }
+
+      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      // this is the main constructor that does everything
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            // this si where the group by aggregation happens
+            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
+      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
+
+      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+
+      private void _main_stream_onchange(object nv) {
+         DataPacket dp = nv as DataPacket;
+         ///grouping together
+         _timer.Restart();
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         _timer.Stop();
+         var groupElapsedMs = _timer.ElapsedMilliseconds;
+         if (groupElapsedMs >= 50) {
+            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
+         }
+         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         /// reducing
+         _timer.Restart();
+         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         _timer.Stop();
+         var applyElapsedMs = _timer.ElapsedMilliseconds;
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
+            _n_expensive_fires++;
+            Log.DataFlow.info(
+               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+      }
+   }
+}
@@ -0,0 +1,187 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using RowHash = System.Int32;
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   using System.Collections;
+   using System.Data.Entity.Core.Mapping;
+
+   [DataStreamUserRespresentation("JoinStream")]
+   public class SimpleJoinStream : ViewStream {
+      public class JoinBookkeepingData {
+         // initialized in cons
+         private readonly List<string> _join_col_names;
+         private readonly bool _is_left;
+         private readonly BlueRowHasher _hasher;
+         // initialized on first udpate         
+         private IBlueTableSchema _schema;
+         private int[] _join_cols;
+         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
+         
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
+            _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
+            _is_left = is_left;
+            _hasher = row_hasher;
+         }
+
+         public IBlueTableSchema schema => _schema;
+
+         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
+
+         public List<IBlueRow> get_matching_rows (int hash) {
+            List<IBlueRow> rv = null;
+            _hash_row_map.TryGetValue(hash, out rv);
+            return rv;
+         }
+
+         private RowHash _get_hash(IBlueRow r) {
+            if (_join_cols == null || _join_cols.Length == 0) { 
+               return 0;
+            }
+            return _hasher.get_hash(r, _join_cols);
+         }
+
+         public void update_data(DataPacket dp) {
+            if (!ready) {               
+               _schema = dp.newRows.schema;
+               _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+            }
+            _hash_row_map.Clear();
+            foreach (IBlueRow r in dp.newRows) {
+               int hash = _get_hash(r);
+               List<IBlueRow> hash_rows = null;
+               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
+                  hash_rows = new List<IBlueRow>();
+                  _hash_row_map.Add(hash, hash_rows);
+               }
+               hash_rows.Add(r);
+            }
+         }
+
+         public bool ready {
+            get {
+               return _schema != null;
+            }
+         }
+
+      }
+
+      private ViewStream _ls, _rs;
+      private JoinBookkeepingData _ldata, _rdata;
+      private ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema = null;
+
+      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
+
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
+         _ls = ls;
+         _rs = rs;
+         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
+         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
+         Log.assert(lj_col_list.Count ==  rj_col_list.Count);
+         int n_cols = lj_col_list.Count;
+         BlueRowHasher hasher = new BlueRowHasher(n_cols);
+         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
+         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _ls.onChange += _ls_onchange;
+         _rs.onChange += _rs_onchange;
+      }
+
+      private void _ls_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _ldata.update_data(dp);
+         _form_and_fire();
+      }
+
+      private void _rs_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket ;
+         _rdata.update_data(dp);
+         _form_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         _ls.onChange -= _ls_onchange;
+         _rs.onChange -= _rs_onchange;
+      }
+
+      private void _build_schema() {
+         if (_schema != null)
+            return;
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
+         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
+            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
+            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
+         }
+         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
+            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
+            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
+         }
+         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
+      }
+
+      private void _form_and_fire() {
+         if (!_ldata.ready||!_rdata.ready)
+            return;
+         _build_schema();
+         List<object[]> res_rows = new List<object[]>();
+         
+         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
+            int offset = 1;
+            JoinBookkeepingData sdata = _ldata;
+            if (is_left) {
+               offset = 1;
+               sdata = _ldata;
+            } else {
+               sdata = _rdata;
+               offset = 1 + _ldata.schema.columnInfo.Length;
+            }
+            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
+               rval[offset + i] = r[i];
+            }
+         };
+         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
+            object[] rv = new object[_schema.columnNames.Length];            
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
+               rv[i] = DBNull.Value;
+            }
+            rv[0] = id;
+            copy_row(rv, l_row, true);
+            return rv;
+         };
+         // this is where the join happens
+         foreach (var kv in _ldata.rows) {
+            RowHash hash = kv.Key;
+            List<IBlueRow> l_rows = kv.Value;
+            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
+
+            foreach (IBlueRow lr in l_rows) {                              
+               if (r_rows != null) {
+                  foreach (IBlueRow r in r_rows) {
+                     object[] rowvals = build_row_array(res_rows.Count, lr);
+                     copy_row(rowvals, r, false);
+                     res_rows.Add(rowvals);
+                  }
+               } else {
+                  object[] rowvals = build_row_array(res_rows.Count, lr);
+                  res_rows.Add(rowvals);
+               }
+            }
+         }
+         BlueTable rv = new BlueTable(_schema);
+         foreach (object[] rval in res_rows) {
+            rv.addRow(new BlueRow(_schema, rval));
+         }
+         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+   }
+}
@@ -0,0 +1,28 @@
+//namespace bts.Blueorb {
+//   using BCG;
+
+//   public class JsonSaverStream : ViewStream {
+//      private ViewStream _stream;
+//      private string _file_name;
+//      public JsonSaverStream(ViewStream vs, string file_name) {
+//         _stream = vs;
+//         _file_name = file_name;
+//      }
+
+//      protected override void myOnFirstSubscriber() {
+//         _stream.onChange += _on_data;
+//      }
+
+//      private void _on_data(object dp_obj) {
+//         if (dp_obj == null) {
+//            return;
+//         }
+//         DataPacket dp = dp_obj as DataPacket;
+//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
+//      }
+
+//      protected override void myOnLastSubscriber() {
+//         _stream.onChange -= _on_data;
+//      }
+//   }
+//}
@@ -0,0 +1,91 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+using Amazon.Runtime.Internal.Util;
+
+using BCG;
+using BCG.Utilities;
+
+using bts;
+
+namespace bts.BlueOrb{
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      // col_name: col_value ... ; col_name: col_value
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');
+
+
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
+         int n = 1;
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
+            if (columnAndType.Length == 1)
+               ci[n] = new ColumnInfo(c, n, typeof(double));
+            else
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
+            n++;
+         }
+
+
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
+         n = 0;
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
+                  continue;
+               t.addRow(
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
+               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
+            }
+
+         mySetRealSource(new BlueRowCollectionStaticStream(t));
+      }
+   }
+}
@@ -0,0 +1,89 @@
+using System;
+
+using BCG.Utilities.General;
+using BCG;
+using bts;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("MillisecondStream")]
+   public class MillisecondStream : ViewStream, IWorker {
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private readonly TimeSpan _period;
+      private readonly int _size;
+      private readonly ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema;
+      private bts.CircularBufferTable _table;
+
+      public MillisecondStream(int interval_msecs, int size)
+      : this(ViewStreamDatabase.instance, interval_msecs, size) {
+      }
+
+      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
+         _vsd = vsd;
+         _period = TimeSpan.FromMilliseconds(interval_msecs);
+         _size = size;
+         _schema = BlueDataUtility.form_schema("Milliseconds",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("time", TypeCode.DateTime),
+               Tuple.Create("display_time", TypeCode.String));
+         _table = new CircularBufferTable(_schema, _size);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private int _id = 0;
+      public void doWork() {
+         if (DateTime.Now - _last_fire_time >= _period) {
+            _last_fire_time = DateTime.Now;
+            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
+            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
+            _table.add_row(nr);
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+   }
+   internal class MSecondStream : TimeUnitStream {
+      public MSecondStream(int intervalInms, int size)
+         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
+
+      protected override DateTime myRoundTime(DateTime dt) { return dt; }
+   }
+
+   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
+                                                               new ColumnInfo("id", 0, typeof(int)),
+                                                               new ColumnInfo("time", 1, typeof(string)));
+      public DisplayableMillisecondStream(int intervalInMs, int sz) {
+         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
+      }
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row) {
+         DateTime time = row.get_datetime("time");
+         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+         return new BlueRow(mySchema, row["id"], time_txt);
+      }
+
+      #endregion
+
+      private string myToString(DateTime dt) {
+         string sep = ":";
+         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
+      }
+   }
+}
@@ -0,0 +1,161 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   //using MathNet.Numerics.Statistics;
+
+
+   public enum MultiRowOp {
+      Mean,
+      Median,
+      Mode,
+      Range,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+   }
+   public class MultiRowOpHelper {
+      // input + op
+      public readonly List<Tuple<string, MultiRowOp>> operations;
+
+      public static MultiRowOpHelper parse_ops(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new MultiRowOpHelper(rv);
+      }
+
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+         operations = operations_;
+      }
+
+      // aggregates multiple rows into one - ie does group_by compute
+      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      // aggregates multiple rows into one, used by MultiRowRollingMathStream
+      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rows.ElementAt(start_idx);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         // out_col vs inp_cal + type
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+            or.setValue("id", ir["id"]);
+         }
+         return or;
+      }
+
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+         switch (op) {
+            case MultiRowOp.Max:
+               return col_vals.Max<double>();
+            case MultiRowOp.Min:
+               return col_vals.Min<double>();
+            case MultiRowOp.Mean:
+               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
+            case MultiRowOp.Sum:
+               return col_vals.Sum();
+            case MultiRowOp.Median:
+               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
+            //case MultiRowOp.Mode:
+            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
+            case MultiRowOp.First:
+               return col_vals.First();
+            case MultiRowOp.Last:
+               return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}
@@ -0,0 +1,40 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System.Collections.Generic;
+   using System.Net;
+
+   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
+   public class MultiRowRollingMathStream : ViewStream {
+      private ViewStream _sorted_parent_str;
+      private MultiRowOpHelper _op;
+      private int _rolling_window_sz;
+      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
+         _sorted_parent_str = new SortStream(parent_str, sort_exp);
+         _op = MultiRowOpHelper.parse_ops(op_txt);
+         _rolling_window_sz = rolling_window_sz;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _sorted_parent_str.onChange += _parent_str_onchange;
+      }      
+
+      protected override void myOnLastSubscriber() {
+         _sorted_parent_str.onChange -= _parent_str_onchange;
+      }
+
+      private void _parent_str_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         var rows = dp.newRows.to_list();
+         BlueTable out_table = null;
+         for (int i = 0; i < rows.Count; i++) {
+            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
+            if (out_table == null)
+               out_table = new BlueTable(out_r.schema);
+            out_table.addRow(out_r);
+         }            
+         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
+         myFireEvent(dp_out);
+      }
+   }
+}
@@ -0,0 +1,188 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   #region mappings
+   internal class ProjectData {
+      // schema from the base stream
+      public readonly IBlueTableSchema in_schema;
+      // this is the projected schema
+      public readonly IBlueTableSchema out_schema;
+      // projected cols to the input cols, for cols who's names are changed
+      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
+      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
+                  IDictionary<string, string> mapped_to_orig_col_map_) {
+         in_schema = in_schema_;
+         out_schema = out_schema_;
+         mapped_to_orig_col_map = mapped_to_orig_col_map_;
+      }
+   }
+   #endregion
+
+
+#region Row with mappings build in
+   internal class NewProjectRow : IBlueRow {
+      #region IBlueRow members
+      private ProjectData _project_data;
+      private IBlueRow _row;
+      public NewProjectRow(ProjectData project_data, IBlueRow row) {
+         _project_data = project_data;
+         _row = row;
+      }
+      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
+      #endregion
+
+      #region ICollection members
+      public int Count { get { return schema.columnNames.Length; } }
+
+      public bool IsSynchronized { get { return false; } }
+
+      public object SyncRoot { get { return this; } }
+
+      public void CopyTo(Array array, int index) {
+         int i = index;
+         foreach (object o in this as IEnumerable)
+            array.SetValue(o, i++);
+      }
+      #endregion
+
+      #region IDictionary members
+      public bool IsFixedSize { get { return true; } }
+
+      public bool IsReadOnly { get { return true; } }
+
+      public object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            return this[(int)key];
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public object this[int index] {
+         get {
+            return this[schema.columnInfo[index].colName];
+         }
+      }
+
+      public object this[string col_name] {
+         get {
+            string mapped_to_col = null;
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+               return _row[mapped_to_col];
+            }
+            return _row[col_name];
+         }
+      }
+
+      public ICollection Keys { get { return schema.columnNames; } }
+
+      public ICollection Values {
+         get {
+            object[] retVal = new object[schema.columnNames.Length];
+            CopyTo(retVal, 0);
+            return retVal;
+         }
+      }
+
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public bool Contains(object key) { return schema[(string)key] != null; }
+
+      /// <summary>
+      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+      /// iterate through the values, typecast this into an IEnumerable
+      /// </summary>
+      /// <returns></returns>
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      #endregion
+   }
+   #endregion
+
+   [DataStreamUserRespresentation("ProjectStream")]
+   public class NewProjectStream : ViewStream, IStreamMapper {
+
+      private ViewStream _stream;
+      private string _project_expression;
+      private ProjectData _project_data = null;
+      //public ProjectData 
+      public NewProjectStream(ViewStream vs, string project_expression) {
+         _stream = vs;
+         _project_expression = project_expression;
+         _stream = new MapStream(vs, this);
+         mySetRealSource(_stream);
+      }
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+         _project_data = _create_project_data(input_schema, _project_expression);
+         return _project_data.out_schema;
+      }
+
+      public IBlueRow map(IBlueRow row) {
+         return new NewProjectRow(_project_data, row);
+      }
+
+      // this is where most of the logic is, in terms of translating cols
+      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
+                                                      string project_expression) {
+         project_expression = project_expression.Trim();
+         string[] project_toks = project_expression.Split(' ', ',');
+         HashSet<string> removed_cols = new HashSet<string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // modifications (6/30/2025):
+         // only the columns represented here, either mapped
+         // or not, will show
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa b") -> aa b
+         foreach (string o_p_tok in project_toks) {
+            string p_tok = o_p_tok.Trim();
+            if (p_tok.Contains(":")) {
+               // explicit mapping
+               string[] map_toks = p_tok.Split(":");
+               string orig_col = map_toks[0];
+               string mapped_col = map_toks[1];
+               orig_to_mapped.Add(orig_col, mapped_col);  
+            } else {
+               // this is a col_name explicitely stated
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
+            }
+         }
+
+         // make the new schema
+         List<ColumnInfo> new_cols = new List<ColumnInfo>();
+         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;           
+            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
+               string mapped_name = orig_to_mapped[ci.colName];
+               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            }
+         }
+         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
+         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
+      }
+
+   
+
+   }
+}
@@ -0,0 +1,127 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using System.IO;
+   using System.Diagnostics;
+   using System;
+
+   [DataStreamUserRespresentation("PythonDataframeStream")]
+   public class PythonDataframeStream : ViewStream {
+      private ViewStream _parent_str;
+      private readonly string _in_csv_file_name;
+      private readonly string _out_csv_filename;
+      private readonly string _transform_function;
+      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
+         var guid = System.Guid.NewGuid();
+         _in_csv_file_name = $"pdfs_in_{guid}.csv";
+         _out_csv_filename = $"pdfs_out_{guid}.csv";
+         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
+         _transform_function = transform_function;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _parent_str.onChange += _parent_str_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _parent_str.onChange -= _parent_str_onchange;
+      }
+
+      private void _parent_str_onchange(object dp_obj) {
+         Log.assert(dp_obj != null);
+         // the file _in_csv_file_name is now populated.
+         // call py_func_wrapper_main and pass this file name as well
+         // as the output file name
+         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
+         // transform function, which returns the tranformed df. py_func_wrapper_main then
+         // writes it into the _out_csv_filename file.
+         // now this function should read _out_csv_filename into a dp
+         // and fire it
+         _compute_and_write();
+         ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
+         IBlueRowCollection nrows = null;
+         rs.onChange += (dp_obj) => {
+            nrows = (dp_obj as DataPacket).newRows;
+         };
+         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      private void _compute_and_write() {
+         File.WriteAllText("user_code.py", _transform_function);
+         // wrapper function
+         string py_func_wrapper_main = @"
+import sys
+from user_code import transform
+import pandas as pd
+
+if __name__ == '__main__':
+   input_file_name = sys.argv[1]
+   output_file_name = sys.argv[2]
+# 1. load input_file_name into df
+# 2. do some test calc
+# 3. write the output of 2 into output_file_name
+   df = pd.read_csv(input_file_name)   
+   df_out = transform(df)
+   df_out.to_csv(output_file_name, index=False)";
+
+
+         File.WriteAllText("runner.py", py_func_wrapper_main);
+
+         ProcessStartInfo psi = new ProcessStartInfo {
+            FileName = "python",
+            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
+            RedirectStandardOutput = false,
+            UseShellExecute = false
+         };
+         var proc = Process.Start(psi);
+         proc.WaitForExit();
+
+      }
+
+      #region Unused test code
+      private void _simple_test() {
+
+         // actual function that needs to be called ////////////////////////
+         string py_real_function = @"
+def test(x, out_path):
+   rv = x * 10
+   with open(out_path, 'w') as f:
+      f.write(str(rv))
+   return rv";
+         // the function is in a string object.
+         File.WriteAllText("user_code.py", py_real_function);
+
+
+         // runner app (top level main function) //////////////////////////////
+         string py_func_wrapper_main = @"
+import sys
+from user_code import test
+
+if __name__ == '__main__':
+   arg = int(sys.argv[1])
+   out_path = sys.argv[2]
+   rv = test(arg, out_path)
+   rv = rv + 12
+   with open(out_path, 'w') as f:to \
+      f.write(str(rv))";
+   
+         File.WriteAllText("runner.py", py_func_wrapper_main);
+
+         string outputPath = "py_out.txt";
+         ProcessStartInfo psi = new ProcessStartInfo {
+            FileName = "python",
+            Arguments = $"runner.py 4 {outputPath}",
+            RedirectStandardOutput = false,
+            UseShellExecute = false
+         };
+         var proc = Process.Start(psi);
+         proc.WaitForExit();
+
+         // Now read the output written by the function
+         string result = File.ReadAllText(outputPath);
+         Console.WriteLine(result); // Should print "6"
+
+      }
+      #endregion
+   }
+}
@@ -0,0 +1,30 @@
+using BCG;
+
+namespace bts.BlueOrb {
+   public class SchemaGuesserStream : ViewStream {
+
+      private IBlueRowCollection _rows = null;
+      private ViewStream _vs;
+      public SchemaGuesserStream(ViewStream vs) {
+         _vs = vs;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vs.onChange += _on_data;
+      }
+      protected override void myOnLastSubscriber() {
+         _vs.onChange -= _on_data;
+      }
+
+      private void _on_data(object obj) {
+         DataPacket dp = obj as DataPacket;
+         if (_rows == null) {
+            _rows = BlueDataUtility.get_typed_table(dp.newRows);
+         }
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+
+      }
+
+   }
+}
@@ -0,0 +1,67 @@
+using Bts.ArithmeticExpressions;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG;
+
+namespace bts.BlueOrb {
+
+   [DataStreamUserRespresentation("SingleRowMathStream")]
+   public class MathStream : ViewStream, IStreamMapper {
+      private IArithmeticProgram myExp;
+      private BooleanExpressionFilter _bool;
+      List<string> myExtraColumns = new List<string>();
+      BlueTableSchema mySchema;
+
+
+      public MathStream(ViewStream vs, string mathExpression)
+         : this(vs, mathExpression, string.Empty) {
+      }
+
+      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
+         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
+            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
+         myExtraColumns = myExp.OutputIdentifiers.ToList();
+         if (!string.IsNullOrEmpty(boolean_expression))
+            _bool = new BooleanExpressionFilter(boolean_expression);
+         mySetRealSource(new MapStream(vs, this));
+      }
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
+            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
+            foreach (ColumnInfo ci in inputSchema.columnInfo)
+               colIdx[ci.colName] = ci;
+
+            foreach (string ncn in myExtraColumns) {
+               ColumnInfo newC;
+               if (colIdx.ContainsKey(ncn))
+                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
+               else
+                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
+               colIdx[ncn] = newC;
+            }
+            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
+         }
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row) {
+         MutableBlueRow result = new MutableBlueRow(mySchema, row);
+         if (_bool == null || _bool.filter(row)) {
+            myExp.Evaluate(result.AsDictionary());
+         }
+         return result;
+      }
+
+      #endregion
+   }
+}
@@ -0,0 +1,17 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   [DataStreamUserRespresentation("SortStream")]
+   public class SortStream: ViewStream {
+      public SortStream (ViewStream parent_stream, string sort_expr) {
+         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,16 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
+   public class TimeBasedRollingFilterStream : ViewStream {
+      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
+         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -1,9 +1,8 @@
-using Google.Protobuf.WellKnownTypes;
-
 using System;
-using BCG.Utilities.General;

-namespace BCG {
+using BCG;
+
+namespace bts.BlueOrb {
   // Keeps adding changing rows to the stream, hence preserving the history
   // and creating a time series, for example this can be used to make bbo updates
   // into a time series of prices/qty 
@@ -17,10 +16,10 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
+         _stream.onChange += _onstream_change;
      }
      protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
+         _stream.onChange -= _onstream_change;
      }


@@ -41,7 +40,6 @@ namespace BCG {
         }
         _table.addRows(added_rows);
         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
-
      }
   }

@@ -0,0 +1,11 @@
+namespace bts.BlueOrb {
+   using BCG;
+   [DataStreamUserRespresentation("TopStream")]
+   public class TopStream : ViewStream { 
+      public TopStream(ViewStream vs, int n_rows, string sort_col) {
+         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -0,0 +1,16 @@
+namespace bts.BlueOrb {
+   using System;
+   using System.Collections.Generic;
+
+   using mbts;
+   using BCG;
+
+   [DataStreamUserRespresentation("UnionStream")]
+   public class UnionStream : ViewStream {
+      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
+         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -18,8 +18,13 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="IronPython" Version="3.4.2" />
+    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
  </ItemGroup>

 </Project>
@@ -2,15 +2,27 @@ namespace Bts.DataStream;

 using BCG;

+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using Microsoft.Extensions.Logging;
+
 using Spark.Core.Disposables;

 using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
 using System.Reactive.Linq;

 public static class DataStreamExtensions {
   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
      source as ViewStream ?? new ObservableViewStream(source);

+   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
+      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+
   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);

@@ -23,4 +35,100 @@ public static class DataStreamExtensions {
               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
      }
   }
+
+   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
+      source.Select(dp => {
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+
+         IBlueRowCollection<T> CastRows(ICollection rows) =>
+            new BlueTableSelectorWrapper<T>(
+               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
+      });
+
+   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
+      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
+      source.AsTables<T, int>(valueSelector, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
+      source.AsTables<T, int>(valueSelector, keySelector);
+
+   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
+      rowKeySelector ??= r => keySelector(valueSelector(r));
+      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
+         var bd = res.to_builder();
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         foreach (IBlueRow r in addedRows) {
+            bd.add_or_update(valueSelector(r));
+         }
+         foreach (IBlueRow r in deletedRows) {
+            bd.remove(rowKeySelector(r));
+         }
+         return bd.to_table();
+      });
+   }
+
+   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.Scan(
+         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
+         (res, table) => {
+            var bd = res.Table.ToDataPacketBuilder();
+            foreach (var r in table.removed) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in table.added) {
+               bd.AddOrUpdate(r);
+            }
+            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
+            return (newRows, resDp);
+         })
+         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
+         .Select(t => t.Dp);
+
+   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.AsDataPackets(schema).AsViewStream();
+
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
+      var columns = schema.columnInfo;
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
+         }
+      }
+      return indices;
+   }
 }
@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream {
+      private readonly string _dsl;
+      public DSLStream(string dsl) {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         _dsl = dsl;
+         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(string stream_expression) {
+         try {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         } catch (Exception ex) {
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}
@@ -4,14 +4,21 @@ using BCG;

 using bts;

+using Microsoft.Extensions.Logging;
+
 using System;

 public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
   readonly IObservable<DataPacket> source;
+   readonly string? name;
+   readonly ILogger? logger;
   IDisposable? subscription;

-   public ObservableViewStream(IObservable<DataPacket> source) =>
+   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
      this.source = source;
+      this.name = name;
+      this.logger = logger;
+   }

   public void Dispose() {
      GC.SuppressFinalize(this);
@@ -23,7 +30,13 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos

   protected override void myOnFirstSubscriber() {
      Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
+      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
+
+      void OnError(Exception exception) =>
+         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
+
+      void OnCompleted() =>
+         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
   }

   protected override void myOnLastSubscriber() {
@@ -25,12 +25,12 @@ namespace BCG {
         byte[] rem_req_bytes = rem_req.serialize();
         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
         _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
+         _source_str.onChange += _handle_data;
      }

      protected override void myOnLastSubscriber() {
         //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
+         _source_str.onChange -= _handle_data;
      }

      static string get_expression(params string[] lines) {
@@ -1,33 +1,51 @@
 using System;
+using System.Collections.Generic;

-namespace BCG {
+namespace BCG {   
   public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_exp(string exp) {
+      public static Tuple<string, Type, object> parse_single_tok(string exp) {
         string[] toks = exp.Split(':');
-         string col_name = toks[0];
-         string col_value = toks[1];
+         string col_name = toks[0].Trim();
+         string col_value = toks[1].Trim();
         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
         return Tuple.Create(col_name, tup.Item1, tup.Item2);
      }

+      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
+         string[] toks = exp.Split(",");
+         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
+         foreach (string s in toks) {
+            var tup = parse_single_tok(s.Trim());
+            rv.Add(tup);
+         }
+         return rv;
+      }
+
      private Tuple<string, Type, object> _new_col;
+      private List<Tuple<string, Type, object>> _new_col_map;
      private IBlueTableSchema _out_schema;

      public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col = parse_exp(col_def);
+         _new_col_map = parse_exp(col_def);
         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
         mySetRealSource(rs);
      }

      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null)
-            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
+         if (_out_schema == null) {
+            _out_schema = input_schema;
+            foreach (var m in _new_col_map) {
+               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
+            }
+         }
         return _out_schema;
      }

      private IBlueRow _map(IBlueRow r) {
         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         rv.setValue(_new_col.Item1, _new_col.Item3);
+         foreach (var m in _new_col_map) {
+            rv.setValue(m.Item1, m.Item3);
+         }
         return rv;
      }
   }
@@ -1,52 +1,45 @@
 using System;

-namespace BCG
-{
-   [DataStreamUserRespresentation ("count")]
-	public class CountStream : ViewStream
-	{
+namespace bts.BlueOrb {
+   using BCG;
+   [DataStreamUserRespresentation("CountStream")]
+   public class CountStream : ViewStream {
      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
-         new ColumnInfo ("id", 0, typeof(int)),
-         new ColumnInfo ("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
      private int myId = 0;
      private ViewStream myVS;

-      public CountStream (ViewStream vs)
-      {
+      public CountStream(ViewStream vs) {
         myVS = vs;
      }

-      public CountStream (string tableName)
-         : this (new TableStream (tableName))
-      {}
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }

-      protected override void myOnFirstSubscriber () {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber () {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
+      protected override void myOnFirstSubscriber () =>
+         myVS.onChange += myHandleChanges;

+      protected override void myOnLastSubscriber () =>
+         myVS.onChange -= myHandleChanges;

-      private void myHandleChanges(object newValue)
-      {
-         myFireEvent (
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
            myLastFiredDataPacket =
               myLastFiredDataPacket == null ?
-                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
-                  new BlueTable (mySchema))
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
                  :
-                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
                  myLastFiredDataPacket.newRows)
            );
      }

-      private IBlueRowCollection myGetRows (object dp, int id)
-      {
-         BlueTable retVal = new BlueTable (mySchema);
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
         retVal.addRow
-            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
         return retVal;
      }

@@ -30,7 +30,7 @@ namespace BCG

        protected override void myOnFirstSubscriber()
        {
-            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
+            myToCrush.onChange += myToCrush_onChange;
        }

        protected override void myOnLastSubscriber()
@@ -15,11 +15,11 @@ namespace BCG
            
            //set up an anonymous delegate for each of the streams
            //we do this so that we can unsubscribe
-            myDelegates = new ChangeDelegate[myToCrush.Length];
+            myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
            for (int i = 0; i < myToCrush.Length; ++i)
            {
                int tempI = i;
-                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
+                myDelegates[tempI] = o => myHandleChange(tempI, ((DataPacket)o).newRows);
            }
        }

@@ -27,7 +27,7 @@ namespace BCG
        private ViewStream[] myToCrush;
        private ITableCrusher myCrusher;

-        private ChangeDelegate[] myDelegates;
+        private DataStreamChangeDelegate[] myDelegates;

        protected override void myOnFirstSubscriber()
        {
@@ -28,7 +28,7 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myDataSource.onChange += new ChangeDelegate(myDataSource_onChange);
+         myDataSource.onChange += myDataSource_onChange;
      }

      protected override void myOnLastSubscriber()
@@ -18,7 +18,8 @@ namespace bts {
         _filename = filename;
      }

-      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
+      protected override void myOnFirstSubscriber() =>
+         _stream.onChange += _handle_changes;

      void _handle_changes(object dp_obj) {
         DataPacket dp = dp_obj as DataPacket;
@@ -26,7 +27,8 @@ namespace bts {
         myFireEvent(dp_obj);
      }

-      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
+      protected override void myOnLastSubscriber() =>
+         _stream.onChange -= _handle_changes;

      protected void _save(DataPacket dp) {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
@@ -8,25 +8,20 @@ using mbts.config;
 using System;
 using System.Text.RegularExpressions;

-namespace BCG.StreamWorld.Addition
-{
-   [DataStreamUserRespresentation (false)]
-   public class FilterStream : ViewStream
-   {
+namespace BCG.StreamWorld.Addition {
+   [DataStreamUserRespresentation(false)]
+   public class FilterStream : ViewStream {
      private ViewStream myBaseStream;
      private BlueTable myBaseTable, myStateTable;
      private IStreamFilter myFilter;

      public FilterStream(ViewStream vs)
-         : this(vs, "1=1")
-      { }
+         : this(vs, "1=1") { }

      public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp))
-      { }
+         : this(vs, new BooleanExpressionFilter(filterExp)) { }

-      public FilterStream(ViewStream vs, IStreamFilter filter)
-      {
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
         myBaseStream = vs;
         myFilter = filter;
         vs.ErrorChanged += OnError;
@@ -35,14 +30,12 @@ namespace BCG.StreamWorld.Addition
      }


-      protected override void myOnFirstSubscriber()
-      {
-         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
-      }
+      protected override void myOnFirstSubscriber() =>
+         myBaseStream.onChange += myHandleChanges;

      protected override void myOnLastSubscriber()
      {
-         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
+         myBaseStream.onChange -= myHandleChanges;
         myBaseTable.clear();
         myStateTable.clear();
      }
@@ -57,23 +50,19 @@ namespace BCG.StreamWorld.Addition
         return false;
      }

-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
         try {
            DataPacket dp = newValue as DataPacket;
-            if (myStateTable == null)
-            {
+            if (myStateTable == null) {
               bts.Log.assert(myBaseTable == null);
               myStateTable = new BlueTable(dp.schema);
               myBaseTable = new BlueTable(dp.schema);
            }
            BlueTable dels = new BlueTable(dp.schema),
                        adds = new BlueTable(dp.schema);
-            foreach (IBlueRow r in dp.deletedRows)
-            {
+            foreach (IBlueRow r in dp.deletedRows) {
               myBaseTable.deleteRow(r);
-               if (myStateTable.containsID((int)r["id"]))
-               {
+               if (myStateTable.containsID((int)r["id"])) {
                  dels.addRow(r);
                  myStateTable.deleteRow(r);
               }
@@ -85,21 +74,17 @@ namespace BCG.StreamWorld.Addition
                  myStateTable.addRow(r);
               }
            }
-            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null)) {
+            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
               myFireEvent(new DataPacket(myStateTable, adds, dels));
-            }
         } catch (Exception ex) {
            OnError(ex);
         }
      }

-      public IStreamFilter filter
-      {
-         set
-         {
+      public IStreamFilter filter {
+         set {
            myFilter = value;
-            if (myBaseTable != null)
-            {
+            if (myBaseTable != null) {
               IBlueRowCollection oldState = new BlueTable(myStateTable);
               myStateTable.clear();
               // If the filter throws an exception, we can rethrow it as a StreamFilterException
@@ -228,13 +213,13 @@ namespace BCG
      #endregion

      #region overridden methods
-      protected override void myOnFirstSubscriber()
-      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() =>
+         myParentStream.onChange += myHandleCallbacks;

      protected override void myOnLastSubscriber()
      {
         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         myParentStream.onChange -= myHandleCallbacks;
      }

      protected override void myFireEvent(object o)
@@ -245,27 +230,31 @@ namespace BCG
      }
      #endregion

-      private void myHandleCallbacks(object newValue) {
+      private void myHandleCallbacks(object newValue)
+      {
         DataPacket retVal;
         DataPacket arrivedPacket = newValue as DataPacket;
         myLastReceivedRows = arrivedPacket.newRows;
         try {
-            if (myLastFiredRows != null) {
+            if (myLastFiredRows != null)
+            {
               retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
                  new BlueTable(myLastFiredRows.schema));
-               foreach (IBlueRow r in arrivedPacket.deletedRows) {
-                  if (myLastFiredRows.containsID((int)r["id"])) {
+               foreach (IBlueRow r in arrivedPacket.deletedRows)
+                  if (myLastFiredRows.containsID((int)r["id"]))
+                  {
                     myLastFiredRows.deleteRow(r);
                     (retVal.deletedRows as BlueTable).addRow(r);
                  }
-               }
-               foreach (IBlueRow r in arrivedPacket.addedRows) {
-                  if (filterRow(r)) {
+               foreach (IBlueRow r in arrivedPacket.addedRows)
+                  if (filterRow(r))
+                  {
                     myLastFiredRows.addRow(r);
                     (retVal.addedRows as BlueTable).addRow(r);
                  }
-               }
-            } else {
+            }
+            else
+            {
               myLastFiredRows = new BlueTable(arrivedPacket.schema);
               retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
               foreach (IBlueRow r in arrivedPacket.newRows)
@@ -291,19 +280,18 @@ namespace BCG
               myFilter = value;
               if (numberOfSubscribers > 0)
               {
-                  if (myLastReceivedRows != null) {
+                   if (myLastReceivedRows != null)
+                   {
                     try {
-                        BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                        foreach (IBlueRow r in myLastReceivedRows) {
-                           if (filterRow(r)) {
-                              nRows.addRow(r);
-                           }
-                        }
-                        IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                           aRows = new BlueTable(nRows.schema);
-                        BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                        myLastFiredRows = nRows;
-                        myFireEvent(new DataPacket(nRows, aRows, dRows));
+                       BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
+                       foreach (IBlueRow r in myLastReceivedRows)
+                           if (filterRow(r))
+                               nRows.addRow(r);
+                       IBlueRowCollection dRows = new BlueTable(nRows.schema),
+                          aRows = new BlueTable(nRows.schema);                      
+                       BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
+                       myLastFiredRows = nRows;
+                       myFireEvent(new DataPacket(nRows, aRows, dRows));
                     } catch (Exception ex) {
                        OnError(ex);
                     }
@@ -336,8 +324,10 @@ namespace BCG
         try {
            return myFilter.filter(r);
         } catch (Exception ex) {
-            throw new StreamFilterException(StreamId, StreamName, null, ex, r);
+            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
         }
+         return false;
      }
   }
 }
@@ -0,0 +1,24 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name)
+      : this(parent_str, col_name, true) { }
+      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
+         ViewStream rs = new FilterStream(parent_str,
+         r => {
+            bool is_dbnull = r.is_null(col_name);
+            if (is_dbnull) {
+               return false;
+            }
+            if (r.schema[col_name].type_code() == TypeCode.Double) {
+               return !Double.IsNaN(r.get<double>(col_name));
+            }
+            return true;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -52,14 +52,14 @@ namespace BCG {
      }

      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
      }

-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
         myVsd = vsd;
         _time_col = rowTimeColumnName;
         _ttl = rowTimeToLive;
-         myFilterFrequency = filterFrequency;
+         _filter_period = filter_period;
         myNextFilter = DateTime.MinValue;

         // add worker
@@ -92,7 +92,7 @@ namespace BCG {
            // fire event
            onFilterChange?.Invoke();

-            myNextFilter = now.Add(myFilterFrequency);
+            myNextFilter = now.Add(_filter_period);
         }
      }
      #endregion
@@ -108,7 +108,7 @@ namespace BCG {
      private ViewStreamDatabase myVsd;
      private string _time_col;
      private TimeSpan _ttl;
-      private TimeSpan myFilterFrequency;
+      private TimeSpan _filter_period;
      private DateTime myNextFilter;
      #endregion
   }
@@ -36,11 +36,11 @@ namespace BCG
         get{return myIsFrozen;}
      }

-      protected override void myOnFirstSubscriber ()
-      {myStreamToFreeze.onChange += new DataStream.ChangeDelegate (myCallBack);}
+      protected override void myOnFirstSubscriber () =>
+         myStreamToFreeze.onChange += myCallBack;

-      protected override void myOnLastSubscriber ()
-      {myStreamToFreeze.onChange -= new DataStream.ChangeDelegate (myCallBack);}
+      protected override void myOnLastSubscriber () =>
+         myStreamToFreeze.onChange -= myCallBack;

      internal void myCallBack(object o)
      {
@@ -96,15 +96,15 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
-         this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
+         myLeftSource.onChange += myLeft_onChange;
+         myRightSource.onChange += myRight_onChange;
         this.myVsd.addWorker(this);
      }

      protected override void myOnLastSubscriber()
      {
-         this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
-         this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
+         myLeftSource.onChange -= myLeft_onChange;
+         myRightSource.onChange -= myRight_onChange;
         this.myVsd.removeWorker(this);
      }

@@ -1,102 +1,102 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;

-namespace BCG
-{
+// namespace BCG
+// {
   
-    public class AverageGroupsStream : ViewStream
-    {
-        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-        {
+//     public class AverageGroupsStream : ViewStream
+//     {
+//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+//         {
            
-            myColToAverage = colToAverage;
-            myInput = input;
-            myKeyColumn = keyColumn;
-        }
+//             myColToAverage = colToAverage;
+//             myInput = input;
+//             myKeyColumn = keyColumn;
+//         }

-        private ViewStream myInput;
-        private string myColToAverage;
-        private BlueTableSchema mySchema;
-        private string myKeyColumn;
+//         private ViewStream myInput;
+//         private string myColToAverage;
+//         private BlueTableSchema mySchema;
+//         private string myKeyColumn;

-        protected override void myOnFirstSubscriber()
-        {
-            myInput.onChange += new ChangeDelegate(myInput_onChange);
-        }
-        protected override void myOnLastSubscriber()
-        {
-            myInput.onChange -= myInput_onChange;
-        }
-        private void myInput_onChange(object newValue)
-        {
-            if (mySchema == null)
-                myBuildSchema(((DataPacket)newValue).schema);
+//         protected override void myOnFirstSubscriber()
+//         {
+//             myInput.onChange += new ChangeDelegate(myInput_onChange);
+//         }
+//         protected override void myOnLastSubscriber()
+//         {
+//             myInput.onChange -= myInput_onChange;
+//         }
+//         private void myInput_onChange(object newValue)
+//         {
+//             if (mySchema == null)
+//                 myBuildSchema(((DataPacket)newValue).schema);

-            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);

-            if (myLastValue == null)
-                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-            else
-            {
-                IBlueRowCollection added, deleted;
-                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-                myFireEvent(new DataPacket(newRows, added, deleted));
-            }
-        }
+//             if (myLastValue == null)
+//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+//             else
+//             {
+//                 IBlueRowCollection added, deleted;
+//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+//                 myFireEvent(new DataPacket(newRows, added, deleted));
+//             }
+//         }

-        private void myBuildSchema(IBlueTableSchema schema)
-        {
-            List<ColumnInfo> columns = new List<ColumnInfo>();
+//         private void myBuildSchema(IBlueTableSchema schema)
+//         {
+//             List<ColumnInfo> columns = new List<ColumnInfo>();

-            foreach (ColumnInfo ci in schema.columnInfo)
-                columns.Add(ci);
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 columns.Add(ci);

-            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-        }
+//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+//         }



-        private IBlueRowCollection myEvaluate(DataPacket packet)
-        {
+//         private IBlueRowCollection myEvaluate(DataPacket packet)
+//         {
         
-            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-            IDictionary<string, int> colToCount = new Dictionary<string, int>();
-            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
+//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();

-            foreach (IBlueRow r in packet.newRows)
-            {
-                double thisVal = Convert.ToDouble(r[myColToAverage]);
+//             foreach (IBlueRow r in packet.newRows)
+//             {
+//                 double thisVal = Convert.ToDouble(r[myColToAverage]);

-                string keyVal = r[myKeyColumn].ToString();
-                if (!colToCount.ContainsKey(keyVal))
-                {
-                    colToRunningTotal[keyVal] = thisVal;
-                    colToCount[keyVal] = 1;
-                }
-                else
-                {
-                    colToRunningTotal[keyVal] += thisVal;
-                    colToCount[keyVal] += 1;
-                }
-                colToLastRow[keyVal] = r;
-            }
+//                 string keyVal = r[myKeyColumn].ToString();
+//                 if (!colToCount.ContainsKey(keyVal))
+//                 {
+//                     colToRunningTotal[keyVal] = thisVal;
+//                     colToCount[keyVal] = 1;
+//                 }
+//                 else
+//                 {
+//                     colToRunningTotal[keyVal] += thisVal;
+//                     colToCount[keyVal] += 1;
+//                 }
+//                 colToLastRow[keyVal] = r;
+//             }

-            BlueTable result = new BlueTable(mySchema);
-            foreach (string key in colToCount.Keys)
-            {
-                object[] values = new object[mySchema.columnInfo.Length];
+//             BlueTable result = new BlueTable(mySchema);
+//             foreach (string key in colToCount.Keys)
+//             {
+//                 object[] values = new object[mySchema.columnInfo.Length];

-                colToLastRow[key].CopyTo(values, 0);
-                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-                result.addRow(new BlueRow(mySchema, values));
-            }
+//                 colToLastRow[key].CopyTo(values, 0);
+//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+//                 result.addRow(new BlueRow(mySchema, values));
+//             }

-            return result;
-        }
+//             return result;
+//         }

      
-    }
-}
+//     }
+// }
@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("Average")]
+   public class AverageNonIncGroupsStream : ViewStream {
+      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
+         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
+                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
+         mySetRealSource(rs);
+      }
+
+      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
+         IBlueRow inp_row = g_rows.ElementAt(0);
+         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
+         double sum = 0.0;
+         inp_vals.ToList().ForEach(d => sum += d);
+         string out_col_name = $"average_{inp_col_name}";
+         double average = sum / inp_vals.Count();
+         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
+         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
+         out_row.setValue(out_col_name, average);
+         List<IBlueRow> rv = new List<IBlueRow>();
+         rv.Add(out_row);
+         return rv;
+      }
+   }
+}
@@ -0,0 +1,126 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   //using MathNet.Numerics.Statistics;
+
+
+   public enum MultiRowOp {
+      Mean,
+      Median,
+      Mode,
+      Range,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
+   }
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
+
+      public static MultiRowOpHelper parse(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new MultiRowOpHelper(rv);
+      }
+
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+         operations = operations_;
+      }
+
+      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+         switch (op) {
+            case MultiRowOp.Max:
+               return col_vals.Max<double>();
+            case MultiRowOp.Min:
+               return col_vals.Min<double>();
+            case MultiRowOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case MultiRowOp.Sum:
+               return col_vals.Sum();
+            case MultiRowOp.First:
+               return col_vals.First();
+            case MultiRowOp.Last:
+               return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}
@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Linq;

 namespace BCG {
-   public class GroupByStream : MethodDrivenStream {
+   internal class GroupByStream : MethodDrivenStream {
      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
         private string[] _groupby_cols;
         private Func<DataPacket, IBlueRow> _groupby_func;
@@ -175,7 +175,7 @@ namespace BCG {
         //gets resubscribed, we start the process of building data again, and treat the callback as the first 
         //callback.
         myInit();
-         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+         myStream.onChange -= myChangeHandler;
      }

      protected override void myInit() {
@@ -10,6 +10,6 @@ namespace BCG {
        }

        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
    }
 }
@@ -82,12 +82,12 @@ namespace BCG

      protected override void myOnFirstSubscriber() 
      {
-         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
+         myStream.onChange += myChangeHandler;
      }

      protected override void myOnLastSubscriber () 
      {
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myStream.onChange -= myChangeHandler;
      }
   }
 }
@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("AggregationStream")]
+   public class AgregationStream : ViewStream {
+
+      public AgregationStream(ViewStream parent_str, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
+         mySetRealSource(rs);
+      }
+
+      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -1,13 +1,40 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;

 using bts;
+using bts.SilverPine;

-namespace BCG
-{
-   public class NonIncGroupByStream : ViewStream
-   {
+namespace BCG {
+   //[DataStreamUserRespresentation("AggregateStream")]
+   public class NonIncGroupByStream : ViewStream {
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
      private ViewStream _main_stream;
      private INonIncGroupByEvaluator _groupby_func;
      private BlueRowHasher _row_hasher;
@@ -15,24 +42,50 @@ namespace BCG
      private Stopwatch _timer = new();
      private int _n_total_fires = 0, _n_expensive_fires = 0;

-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
-      {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
      }

+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }

-      private void _main_stream_onchange(object nv)
-      {
+      private void _main_stream_onchange(object nv) {
         DataPacket dp = nv as DataPacket;
         ///grouping together
         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
         _timer.Stop();
         var groupElapsedMs = _timer.ElapsedMilliseconds;
         if (groupElapsedMs >= 50) {
@@ -48,14 +101,15 @@ namespace BCG
            return;
         }
         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;         
-         if (applyElapsedMs >= 50)
-         {
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
            _n_expensive_fires++;
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }         
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
      }
   }
 }
@@ -1,31 +1,31 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+//using System;
+//using System.Collections.Generic;
+//using System.Linq;
+//using System.Text;

-namespace BCG
-{
-   public class FullJoinStream : ViewStream
-   {
-      private ViewStream myMainStream;
-      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
-         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
-      {
-         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
-         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
-         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
-         myMainStream = new AppendStream(myMainStream, filtered);
-      }
+//namespace BCG
+//{
+//   public class FullJoinStream : ViewStream
+//   {
+//      private ViewStream myMainStream;
+//      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
+//         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
+//      {
+//         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
+//         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
+//         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
+//         myMainStream = new AppendStream(myMainStream, filtered);
+//      }

-      protected override void myOnFirstSubscriber()
-      {
-         myMainStream.onChange += myHandleMainStreamChange;
-      }
+//      protected override void myOnFirstSubscriber()
+//      {
+//         myMainStream.onChange += myHandleMainStreamChange;
+//      }

-      private void myHandleMainStreamChange(object newData)
-      {
-         IBlueRowCollection result = (newData as DataPacket).newRows;
-         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}
+//      private void myHandleMainStreamChange(object newData)
+//      {
+//         IBlueRowCollection result = (newData as DataPacket).newRows;
+//         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
+//      }
+//   }
+//}
@@ -31,15 +31,15 @@ namespace BCG
      {
         myLeftOperand.master = this;
         myRightOperand.master = this;
-         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
-         myRightStream.onChange += new DataStream.ChangeDelegate (myRightOperand.changeHandler);
+         myLeftStream.onChange += myLeftOperand.changeHandler;
+         myRightStream.onChange += myRightOperand.changeHandler;
      }
      
      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
-         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightOperand.changeHandler);
+         myLeftStream.onChange -= myLeftOperand.changeHandler;
+         myRightStream.onChange -= myRightOperand.changeHandler;
      }

      public JoinOperand getOtherOperand (JoinOperand jo)
@@ -116,7 +116,7 @@ namespace BCG
               IDictionary  map = resolver.getMap ();
               map["id"] = "id";
               myPs = new ProjectStream (myIjs, map);
-               myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
+               myPs.onChange += myProjectChangeHandler;
               return;
            }
            else
@@ -129,26 +129,26 @@ namespace BCG
      private void myProjectChangeHandler (object o)
      {
         if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
-            myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+            myIjs.onChange -= myInternalJoinChangeHandler;
         myFireEvent ((o as ICloneable).Clone());
         
      }

      protected override void myOnFirstSubscriber ()
      {
-         myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myIjs.onChange += myInternalJoinChangeHandler;
      }

      protected override void myOnLastSubscriber ()
      {
         // how does this relate to the -= in myProjectChangeHandler?
-         myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myIjs.onChange -= myInternalJoinChangeHandler;

         // if there is a  project after the ijs we need to 
         // unsubscribe to it otherwise references will be kept and 
         // a memory leak will occur
         if (myPs != null)
-            myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
+            myPs.onChange -= myProjectChangeHandler;
         myPs = null;
      }

@@ -1,15 +1,16 @@
 using bts;
+using bts.collections;

 using mbts;

 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;

-namespace BCG
-{
-   public class NonIncJoinBookkeeper : IDisposable
-   {
+namespace BCG {
+   public class NonIncJoinBookkeeper : IDisposable {
      public Action on_new_data;
      private readonly NonIncJoinSpec _join_spec;
      private readonly int _join_position;
@@ -26,41 +27,71 @@ namespace BCG

      private int[] _join_col_indices = null,
                     _data_col_indices = null;
-      
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
-      {
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
         _join_spec = the_join_spec;
         _join_position = join_position;
         _start_index_in_output = start_index_in_output;
         _row_hasher = row_hasher;
         _is_left_most = _join_position == 0;
         _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }

-      public void _on_stream_cb(object dp_obj)
-      {
+      public void _on_stream_cb(object dp_obj) {
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
-         if (_schema == null)
-         {
+         _fill_missing_data(dp);
+         if (_schema == null) {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
         }

-         if (_is_left_most)
-         {
+         if (_is_left_most) {
            _row_list.Clear();
            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
-         }
-         else
-         {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
            if (_is_tag_along)
               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else
-            {
+            else {
               _row_map.Clear();
               foreach (IBlueRow r in dp.newRows)
                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -79,31 +110,25 @@ namespace BCG

      public bool is_ready { get { return _schema != null; } }

-      public void constribute_cols_to_schema(List<ColumnInfo> cis)
-      {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
      }

      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }

-      public void copy_master_row_to_output_row(int row_index, object[] output_row)
-      {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
         IBlueRow this_row = _row_list[row_index].second;
         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
      }

-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
-      {
-         if (_is_tag_along)
-         {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
            if (_solitary_row != null)
               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
            else
               copy_dbnull_to_output_row(output_row);
-         }
-         else
-         {
+         } else {
            IBlueRow this_row = null;
            if (_row_map.TryGetValue(row_hash, out this_row))
               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -112,8 +137,7 @@ namespace BCG
         }
      }

-      private void copy_dbnull_to_output_row(object[] output_row)
-      {
+      private void copy_dbnull_to_output_row(object[] output_row) {
         for (int i = 0; i < _data_col_indices.Length; i++)
            output_row[i + _start_index_in_output] = DBNull.Value;
      }
@@ -3,63 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+
 using mbts;

-namespace BCG
-{
-   public class NonIncJoinSpec
-   {
+namespace BCG {
+   // this a collection of data, theres no joining logic in here
+   public class NonIncJoinSpec {
      public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

      public NonIncJoinSpec() { }

      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
-      { }
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }

-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
-      {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
         stream = the_stream;
         join_col_names = the_join_col_names;
         data_col_names = the_data_col_names;
         projected_col_names = the_projected_col_names;
      }

-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
-      {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs)
-         {
-            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
            string[] visible_cols = r.third.Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols)
-            {
+            foreach (string s in visible_cols) {
               string[] toks = s.Split(':');
-               if (toks.Length == 1)
-               {
+               if (toks.Length == 1) {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[0]);
-               }
-               else if (toks.Length == 2)
-               {
+               } else if (toks.Length == 2) {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
         }
-         return rv.ToArray ();
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
      }
   }
-
-   //[DataStreamUserRespresentation("ninctest")]
-   //public class NonIncJoinTestStream : ViewStream
-   //{
-   //   public NonIncJoinTestStream (string key_cols, string data_cols)
-   //   {
-   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
-   //   }
-   //}
 }
@@ -6,11 +6,9 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;

-namespace BCG
-{
+namespace BCG {
   [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
-   {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
      private readonly NonIncJoinSpec[] _join_specs;
      private readonly INonIncJoinInlineMapper _output_mapper;
      private readonly BlueRowHasher _row_hasher;
@@ -28,24 +26,19 @@ namespace BCG
               {
                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               })
-      { }
+               }) { }

      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
-      { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }

      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
-      { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }

      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
-      { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }

-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-      {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
         _join_specs = join_specs;
         _output_mapper = output_mapper;
         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -53,12 +46,11 @@ namespace BCG
         _vsd = vsd;
      }

-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++)
-         {
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -66,10 +58,11 @@ namespace BCG
         if (_output_mapper != null)
            _output_mapper.onMapperChange += _on_mapper_change;
         _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            d.Dispose();
         if (_output_mapper != null)
@@ -77,30 +70,26 @@ namespace BCG
         _vsd.removeWorker(this);
      }

-      private void _stream_callback()
-      {
+      private void _stream_callback() {
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
         _is_dirty = true;
         doWork();
      }

-      private void _on_mapper_change()
-      {
+      private void _on_mapper_change() {
         _is_dirty = true;
         doWork();
      }

-      private bool _is_ready()
-      {
+      private bool _is_ready() {
         bool rv = true;
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            rv &= d.is_ready;
         return rv;
      }

-      private void _construct_schema()
-      {
-         if (_output_schema == null)
-         {
+      private void _construct_schema() {
+         if (_output_schema == null) {
            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
               d.constribute_cols_to_schema(cis);
@@ -109,27 +98,23 @@ namespace BCG
      }

      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire()
-      {
-         if (_is_ready())
-         {
+      private void _form_and_fire() {
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
            _construct_schema();
            var timer = Stopwatch.StartNew();
            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
            List<object[]> output_rows = new List<object[]>();

-            for (int i = 0; i < left_row_list.Count; i++)
-            {
+            for (int i = 0; i < left_row_list.Count; i++) {
               object[] or = new object[_output_schema.columnInfo.Length];
               or[0] = i;
               output_rows.Add(or);
            }

-            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
-            {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++)
-               {
+               for (int i = 0; i < left_row_list.Count; i++) {
                  if (ji == 0)
                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                  else
@@ -139,19 +124,20 @@ namespace BCG
            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
            timer.Stop();
            _n_total_fires++;
-            if (timer.ElapsedMilliseconds >= 250) {
-               _n_expensive_fires++;
-               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
-            }
-            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
      }

-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
-      {
-         if (_output_mapper != null)
-         {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
            IBlueRowCollection output_rows;
            var timer = Stopwatch.StartNew();
            output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -160,17 +146,14 @@ namespace BCG
               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
            }
            return output_rows;
-         }
-         else
-         {
+         } else {
            return unmapped_rows;
         }
      }

-      public void doWork()
-      {
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
-         {
+      public void doWork() {
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
            _form_and_fire();
				`@@ -0,0 +1 @@`
				`PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins`