webapp check pt?

refactor: Move logging into its own lib
Also moved a few logging deps from lib_spark -> system: * bts_chrono.h * bts_no_copy.h * defer.h * dispatcher.h/cpp * file_utils.h/cpp * type_name.h
2024-09-18 14:02:40 +00:00 · 2024-09-18 14:02:40 +00:00 · 2024-09-18 14:02:40 +00:00 · 2024-09-18 14:02:40 +00:00 · 2024-09-18 14:02:40 +00:00 · 2024-09-18 14:02:40 +00:00
2076 changed files with 48646 additions and 102173 deletions
@@ -9,7 +9,7 @@
      ]
    },
    "fable": {
-      "version": "4.24.0",
+      "version": "4.0.6",
      "commands": [
        "fable"
      ]
@@ -25,12 +25,6 @@
      "commands": [
        "fake"
      ]
-    },
-    "dotnet-t4": {
-      "version": "3.0.0",
-      "commands": [
-        "t4"
-      ]
    }
  }
-}
+}
@@ -65,7 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7

 # Node
 ARG node_major=18
@@ -129,3 +129,5 @@ RUN \
   -G sudo \
   --password "$(openssl passwd -1 "$docker_pass")" \
   "$docker_user"
+
+RUN usermod -aG docker "$docker_user"
@@ -1,10 +1,15 @@
 {
  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
  "hostRequirements": {
-    "memory": "8gb",
-    "storage": "10gb"
+    "memory": "4gb",
+    "storage": "6gb"
  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
  "updateRemoteUserUID": true,
  "shutdownAction": "none",
  // Note: If running multiple containers on the same host, each must use a
@@ -12,20 +17,26 @@
  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
  "runArgs": [
    "--shm-size",
-    "8gb"
+    "2gb",
+    "--network",
+    "host"
  ],
  "mounts": [
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
  ],
  "customizations": {
    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
      "extensions": [
        "eamodio.gitlens",
        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-python.python",
        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools",
-        "xaver.clang-format"
+        "ms-vscode.cpptools"
      ]
    }
  }
@@ -1,49 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - TODO: pubsub
-#     - TODO: md-publisher
-#     - TODO: webapp (DSQA on the web)
-#     - TODO: api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/server/sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
-# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
-# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
-# SPARKD__DATA_DIR=/home/bts/sparkdata/data
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-
-# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -1,3 +1,4 @@
 *.cmd text=auto eol=crlf
 *.cs text=auto eol=crlf
 *.sh text=auto eol=lf
+*.py text=auto eol=lf
@@ -11,7 +11,6 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
-!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -60,5 +59,3 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
-.env.local
-.secrets
@@ -1,14 +1,6 @@
 {
  "version": "0.2.0",
  "configurations": [
-    {
-      "name": "Python Debugger: Current File with Arguments",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "args": "${command:pickArgs}"
-    },
    {
      "name": ".NET Core Attach",
      "type": "coreclr",
@@ -27,7 +19,7 @@
          "--sparkd",
          "http://127.0.0.1:8000/config"
        ],
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -44,7 +36,7 @@
          "--sparkd",
          "http://127.0.0.1:8000/config"
        ],
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -54,8 +46,8 @@
      "type": "coreclr",
      "request": "launch",
      "preLaunchTask": "Build DataStream WebApp (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "env": {
        "ASPNETCORE_HTTP_PORT": "8080",
        "ASPNETCORE_HTTPS_PORT": "8443",
@@ -67,8 +59,8 @@
      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
      "type": "coreclr",
      "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "env": {
        "ASPNETCORE_HTTP_PORT": "8080",
        "ASPNETCORE_HTTPS_PORT": "8443",
@@ -88,10 +80,10 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
-      "cwd": "build/Debug/bin/net8.0",
      "env": {
        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
      },
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -106,57 +98,13 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
      "env": {
        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
      },
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
-    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "run"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests2",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader2]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
@@ -165,25 +113,12 @@
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "DataStreamQueryAnalyzer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
      },
@@ -202,7 +137,7 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "env": {
        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
      },
@@ -220,7 +155,7 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "env": {
        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
      },
@@ -235,25 +170,11 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
+        "args": [
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -270,40 +191,7 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -312,6 +200,7 @@
      "name": "Sparkd",
      "type": "cppdbg",
      "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
@@ -345,7 +234,7 @@
        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
      },
      "args": [
-        "run"
+        "test-py-md-provider"
      ],
      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
      "stopAtEntry": false,
@@ -403,41 +292,12 @@
        "Staging",
      ]
    },
-    {
-      "id": "dataStreamQueryAnalyzerEnvironment",
-      "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
-      "options": [
-        "Development",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "sparkEnvironment",
-      "type": "pickString",
-      "description": "Spark Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
    {
      "id": "sparkConfigServer",
      "type": "promptString",
      "description": "Config Server URL",
      "default": "http://localhost:8000/config",
    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
    {
      "id": "isSparkApiEnabled",
      "type": "pickString",
@@ -1 +0,0 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
@@ -490,7 +490,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -510,7 +510,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -530,7 +530,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -549,11 +549,12 @@
      "windows": {
        "command": "DataStreamPublisher.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -571,11 +572,12 @@
      "windows": {
        "command": "DataStreamPublisher.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -593,11 +595,12 @@
      "windows": {
        "command": "DataStreamPublisher.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -698,11 +701,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -720,11 +724,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -741,11 +746,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -763,11 +769,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -784,11 +791,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -803,7 +811,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -819,7 +827,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -835,22 +843,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -875,7 +868,7 @@
        "${input:sparkConfigQuery}",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -901,24 +894,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Run Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
    {
      "label": "Run Sparkd Config Query",
      "dependsOn": "Build Sparkd",
@@ -940,65 +915,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Test Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-      "command": "",
-      "windows": {
-        "command": "Spark.Console.exe",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        }
-      },
-    },
-    {
-      "label": "Test Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
-    {
-      "label": "Test Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
    {
      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
      "type": "process",
@@ -1087,30 +1003,26 @@
      "default": "sparkd-debug-clang",
    },
    {
-      "id": "dataStreamPublisherEnvironment",
+      "id": "dataStreamPublisherConfig",
      "type": "pickString",
      "description": "Publisher Configuration",
      "options": [
-        "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
      ],
      "default": "MarketDataPublisher",
    },
    {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
      "type": "pickString",
      "description": "PubSub Configuration",
      "options": [
-        "Development",
-        "EchoServer",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "PubSubServer",
+        "SimpleEchoClient",
      ],
-      "default": "Development",
+      "default": "PubSubServer",
    },
    {
      "id": "sparkConfigQuery",
@@ -1121,16 +1033,6 @@
      "id": "sparkdConfigQuery",
      "type": "promptString",
      "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
+    }
  ]
 }
@@ -1,10 +1,7 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)

-get_version_from_git()
 if(WIN32)
  set(platform "windows")
 else()
@@ -85,15 +82,9 @@ else()
    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
-    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
-
-    # ** Binance Coin Futures **
-    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

    # ** Deribit **
    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -112,15 +103,15 @@ else()
  endif()

  # subdirs
-  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
-    add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance)
-    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/lib_system)
+    add_subdirectory(src/lib_domain)
+    add_subdirectory(src/lib_python_md)
+    add_subdirectory(src/lib_logging)
    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
  endif()
+  add_subdirectory(src/lib_spark)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -24,13 +24,12 @@
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -50,20 +49,10 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
-        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -87,7 +76,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -98,7 +87,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -174,13 +163,12 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_EXPORT_COMPILE_COMMANDS": {
          "type": "BOOL",
          "value": "ON"
        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -218,8 +206,8 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
      }
    },

@@ -251,8 +239,9 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_PROFILES": "default;data_stream",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -305,6 +294,14 @@
      "inherits": "sparkd"
    },

+    {
+      "name": "sparkd-debug-clang-j6",
+      "configurePreset": "sparkd-debug-clang",
+      "displayName": "Sparkd (Debug | Clang | jobs=6)",
+      "inherits": "sparkd",
+      "jobs": 6
+    },
+
    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
@@ -14,34 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -98,102 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -204,19 +80,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-	EndGlobalSection
 EndGlobal
@@ -1,5 +1,3 @@
-include_guard(GLOBAL)
-
 macro(export_to_parent)
  foreach(var IN ITEMS ${ARGN})
    set(${var} ${${var}} PARENT_SCOPE)
@@ -1,48 +0,0 @@
-include_guard(GLOBAL)
-
-function(get_version_from_git)
-  find_package(Git QUIET)
-  if(NOT Git_FOUND)
-    message(WARNING "Git not found")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} describe --tags --always
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    RESULT_VARIABLE GIT_RESULT
-  )
-
-  if(NOT GIT_RESULT EQUAL 0)
-    message(WARNING "Failed to get git tag")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
-
-  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
-  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
-
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
-  else()
-    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
-  endif()
-endfunction()
@@ -1,142 +1,66 @@
 include_guard(DIRECTORY)
 include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)

 function(init_library)
  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
+  set(_one_value_args SOURCE_DIR)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
  if (_ARGS_SHARED AND _ARGS_STATIC)
    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)

-  # Cpp sources
+  set_ternary(_source_dir _ARGS_SOURCE_DIR _ARGS_SOURCE_DIR ${PROJECT_NAME})
+  # Glob cpp sources
  file(
-    GLOB_RECURSE ${_name}_SOURCES
+    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
    ${_source_dir}/*.cpp
  )
  # Note: cpp files inside 'tests' dir(s) are considered tests.
  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
+  foreach(item ${${PROJECT_NAME}_SOURCES})
+    if(${item} MATCHES "tests/*.cpp")
+      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
+      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
    endif()
  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )

  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
+    ${PROJECT_NAME}_SOURCES
+    ${PROJECT_NAME}_TEST_SOURCES
  )

-  # Add main lib
+  # Add main library
  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+    add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES})
  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+    add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+    add_library(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
  endif()
+
  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
+    ${PROJECT_NAME}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
  target_link_directories(
-    ${_name}
+    ${PROJECT_NAME}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )

  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
+  if (${PROJECT_NAME}_TEST_SOURCES)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
    # Catch2
    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
  endif()
 endfunction()
@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()
@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()
@@ -0,0 +1,55 @@
+function(init_shared_library)
+  include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+  set(source_dir)
+  if (${ARGC} EQUAL 0)
+    set(source_dir ${PROJECT_NAME})
+  elseif(${ARGC} GREATER 1)
+    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
+  else()
+    set(source_dir ${ARGV})
+  endif()
+
+  # Glob cpp sources
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${PROJECT_NAME}_SOURCES})
+    if(${item} MATCHES "tests/*.cpp")
+      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
+      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+
+  export_to_parent(
+    ${PROJECT_NAME}_SOURCES
+    ${PROJECT_NAME}_TEST_SOURCES
+  )
+
+  # Add main library
+  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
+  target_include_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  target_link_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+
+  # Add tests executable (if there are any)
+  if (${PROJECT_NAME}_TEST_SOURCES)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction(init_shared_library)
@@ -1,28 +1,19 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(GNUInstallDirs)

 function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
  file(
-    GLOB_RECURSE ${_name}_HEADERS
+    GLOB_RECURSE ${PROJECT_NAME}_HEADERS
    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
+    *.h)

-  target_sources(${_name}
+  target_sources(${PROJECT_NAME}
    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+      FILE_SET HEADERS FILES ${${PROJECT_NAME}_HEADERS})

  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
+    TARGETS ${PROJECT_NAME}
+    EXPORT ${PROJECT_NAME}Targets
    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -30,10 +21,10 @@ function(install_basic)
  )

  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+    EXPORT ${PROJECT_NAME}Targets
+    FILE ${PROJECT_NAME}Targets.cmake
+    NAMESPACE ${PROJECT_NAME}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})

  pack_basic()
 endfunction()
@@ -1,14 +1,6 @@
-include_guard(DIRECTORY)
 include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)

 function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
  find_package(Git QUIET)

  if(GIT_FOUND)
@@ -25,20 +17,20 @@ function(pack_basic)
  endif()

  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
    VERSION ${PROJECT_VERSION}
    COMPATIBILITY AnyNewerVersion
  )

  configure_package_config_file(
    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
  )

  install(
    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
 endfunction()
@@ -1,182 +0,0 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
-set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
-
-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
-      DEPENDS ${ABS_FIL}
-      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
-
-    set(TEMPLATES
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
-    )
-
-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
-      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
-        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
-        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE}
-          ARGS
-            --cpp_out=${_ARGS_CXX_OUT_DIR}
-            --plugin=protoc-gen-service=${_plugin_cmd}
-            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
-            -I ${PROTOBUF_INCLUDE_DIRS}
-            -I ${_protobuf_src_dir}
-            ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_cmd}
-          ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
-    else()
-      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
-        ARGS
-          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
-    endif()
-  endforeach()
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
@@ -2,24 +2,12 @@ include_guard(GLOBAL)

 function(protobuf_generate_cpp)
  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})

  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
  endif()
  set(${_ARGS_CXX_SOURCES})
  set(${_ARGS_CXX_HEADERS})
@@ -38,9 +26,6 @@ function(protobuf_generate_cpp)
    endif()
    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
    add_custom_command(
      OUTPUT
        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
@@ -80,4 +65,3 @@ function(protobuf_generate_cpp)
  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
 endfunction()
-
@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -20,14 +20,16 @@ endfunction()

 function(sym_link_file)
  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
  )
  if (DEFINED _ARGS_INSTALL_DEST)
    install(
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,11 +0,0 @@
-#ifndef BTS_SPARK_VERSION_H
-#define BTS_SPARK_VERSION_H
-
-#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
-#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
-#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
-#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
-#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
-#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
-
-#endif
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -0,0 +1,51 @@
+services:
+  sparkd:
+    build:
+      dockerfile: Dockerfile
+      context: ./sparkd
+    command: [ "./sparkd", "run" ]
+    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
+
+  pubsub:
+    build:
+      dockerfile: pubsub.Dockerfile
+      context: ./DataStream
+    command: [ "./PubSubServer" ]
+    environment:
+      - DOTNET_ENVIRONMENT=Production
+    ports:
+      - 22019:22019
+
+  md-publisher:
+    build:
+      dockerfile: md-publisher.Dockerfile
+      context: ./DataStream
+    depends_on:
+      - pubsub
+      - sparkd
+    command:
+      [
+        "dockerize",
+        "-wait",
+        "tcp://pubsub:22019",
+        "-wait",
+        "tcp://sparkd:8000",
+        "-timeout",
+        "10s",
+        "./DataStreamPublisher"
+      ]
+    environment:
+      - DOTNET_ENVIRONMENT=MarketData
+
+  webapp:
+    ports:
+      - 8080:8080
+      - 8443:8443
+    build:
+      dockerfile: webapp.Dockerfile
+      context: ./DataStream
+    command: [ "./Bts.DataStreamWebApp.Server" ]
+    environment:
+      - ASPNETCORE_HTTP_PORT=8080
+      - ASPNETCORE_HTTPS_PORT=8443
+      - ASPNETCORE_ENVIRONMENT=Staging
@@ -1,100 +1,62 @@
 services:
  sparkd:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
+      dockerfile: src/server/sparkd/Dockerfile
+    command: [ "./sparkd", "run" ]
+    ports: [ "8000:8000", "8001:8001", "8002:8002" ]

-  sparkd-symbols:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+  pubsub:
+    environment:
+      - DOTNET_ENVIRONMENT=Staging
    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/sparkdata/conf/
-        cat /home/bts/sparkdata/conf/.env.local
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
+      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
+      - 22019:22019
+    command: [ "./PubSubServer" ]

-  sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
-    volumes:
-      - type: volume
-        read_only: true
-        source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
-        volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+  md-publisher:
+    build:
+      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+    depends_on:
+      - pubsub
+      - sparkd
+    command:
+      [
+        "dockerize",
+        "-wait",
+        "tcp://pubsub:22019",
+        "-wait",
+        "tcp://sparkd:8000",
+        "-timeout",
+        "10s",
+        "./DataStreamPublisher"
+      ]
+    environment:
+      - DOTNET_ENVIRONMENT=MarketData
+
+  webapp:
+    ports:
+      - 8080:8080
+      - 8443:8443
+    build:
+      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+    command: [ "./Bts.DataStreamWebApp.Server" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
+  api-server:
+    profiles:
+      - api-server
+    depends_on:
+      - sparkd
+    ports:
+      - 8008:8008
+      - 9443:9443
+    build:
+      dockerfile: src/client/spark_api_server/Dockerfile
+    command: [ "./spark_api_server", "run" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging

  sparkd-ping-test:
    depends_on:
@@ -108,56 +70,3 @@ services:
        USER "root"
        RUN apt-get update && apt-get -y install iputils-ping
    command: [ "ping", "sparkd" ]
-
-  pubsub:
-    environment:
-      - DOTNET_ENVIRONMENT=Staging
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
-    ports:
-      - 22019:22019
-    command: [ "./PubSubServer" ]
-
-  md-publisher:
-    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
-    depends_on:
-      - pubsub
-      - sparkd
-    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-
-  webapp:
-    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
-
-volumes:
-  sparkd_data:
-    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, co2::TradingStatus) override;
+virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
 ```

 # Child Orders
@@ -1,38 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts
@@ -24,6 +24,7 @@ RUN locale-gen en_US.UTF-8
 ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
+# ARG entrypoint

 RUN \
  groupadd -g "$docker_uid" "$docker_user" \
@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "6.0.200",
+    "rollForward": "latestMajor"
+  }
+}
@@ -1,42 +1,22 @@
 astroid==2.7.3
 autopep8==1.5.7
-blinker==1.9.0
-boto3==1.37.9
-botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
-click==8.1.8
 colorama==0.4.4
-Flask==3.1.0
 idna==3.4
 isort==5.9.3
-itsdangerous==2.2.0
-Jinja2==3.1.6
-jmespath==1.0.1
 lazy-object-proxy==1.6.0
-MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
 requests==2.31.0
-s3transfer==0.11.4
-six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
-Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -1,164 +1,103 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
-		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
-	EndGlobalSection
-EndGlobal
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		MinSizeRel|x64 = MinSizeRel|x64
+		Release|x64 = Release|x64
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
+EndGlobal
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -41,6 +41,7 @@ Identifier               {Letter}({Letter}*[0-9]*_*)*
 OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
+Negate                   '!'
 MapsTo                   ':'
 IsOfType                 '/'

@@ -93,6 +94,10 @@ IsOfType                 '/'
   return (int)Token.COMMA;
 }

+{Negate} {
+   return (int)Token.NEGATE;
+}
+
 {MapsTo} {
   return (int)Token.MAPS_TO;
 }
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
  </ItemGroup>

  <ItemGroup>
@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

+#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
+#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
        }

+#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
+#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }

-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -12,6 +12,18 @@ module.exports = {
  settings: { react: { version: '18.3.1' } },
  plugins: ['react-refresh'],
  rules: {
+    "import/order": [
+      "error",
+      {
+        "groups": [
+          "builtin",
+          "external",
+          "internal",
+          ["parent", "sibling", "index"]
+        ],
+        "newlines-between": "always"
+      }
+    ],
    'react-refresh/only-export-components': [
      'warn',
      { allowConstantExport: true },
@@ -32,6 +32,7 @@
        "@types/react-dom": "^18.3.0",
        "@vitejs/plugin-react": "^4.3.1",
        "eslint": "^8.57.0",
+        "eslint-plugin-import": "^2.29.1",
        "eslint-plugin-react": "^7.34.2",
        "eslint-plugin-react-hooks": "^4.6.2",
        "eslint-plugin-react-refresh": "^0.4.7",
@@ -3178,6 +3179,12 @@
      "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==",
      "dev": true
    },
+    "node_modules/@types/json5": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
+      "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==",
+      "dev": true
+    },
    "node_modules/@types/parse-json": {
      "version": "4.0.2",
      "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
@@ -3492,6 +3499,26 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/array.prototype.findlastindex": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.5.tgz",
+      "integrity": "sha512-zfETvRFA8o7EiNn++N5f/kaCw221hrpGsDmcpndVupkPzEc1Wuf3VgC0qby1BbHs7f5DVYjgtEU2LLh5bqeGfQ==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.7",
+        "define-properties": "^1.2.1",
+        "es-abstract": "^1.23.2",
+        "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.0.0",
+        "es-shim-unscopables": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
    "node_modules/array.prototype.flat": {
      "version": "1.3.2",
      "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz",
@@ -4306,6 +4333,121 @@
        "url": "https://opencollective.com/eslint"
      }
    },
+    "node_modules/eslint-import-resolver-node": {
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz",
+      "integrity": "sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==",
+      "dev": true,
+      "dependencies": {
+        "debug": "^3.2.7",
+        "is-core-module": "^2.13.0",
+        "resolve": "^1.22.4"
+      }
+    },
+    "node_modules/eslint-import-resolver-node/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "dev": true,
+      "dependencies": {
+        "ms": "^2.1.1"
+      }
+    },
+    "node_modules/eslint-import-resolver-node/node_modules/resolve": {
+      "version": "1.22.8",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
+      "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==",
+      "dev": true,
+      "dependencies": {
+        "is-core-module": "^2.13.0",
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
+      },
+      "bin": {
+        "resolve": "bin/resolve"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/eslint-module-utils": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.1.tgz",
+      "integrity": "sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==",
+      "dev": true,
+      "dependencies": {
+        "debug": "^3.2.7"
+      },
+      "engines": {
+        "node": ">=4"
+      },
+      "peerDependenciesMeta": {
+        "eslint": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/eslint-module-utils/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "dev": true,
+      "dependencies": {
+        "ms": "^2.1.1"
+      }
+    },
+    "node_modules/eslint-plugin-import": {
+      "version": "2.29.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz",
+      "integrity": "sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==",
+      "dev": true,
+      "dependencies": {
+        "array-includes": "^3.1.7",
+        "array.prototype.findlastindex": "^1.2.3",
+        "array.prototype.flat": "^1.3.2",
+        "array.prototype.flatmap": "^1.3.2",
+        "debug": "^3.2.7",
+        "doctrine": "^2.1.0",
+        "eslint-import-resolver-node": "^0.3.9",
+        "eslint-module-utils": "^2.8.0",
+        "hasown": "^2.0.0",
+        "is-core-module": "^2.13.1",
+        "is-glob": "^4.0.3",
+        "minimatch": "^3.1.2",
+        "object.fromentries": "^2.0.7",
+        "object.groupby": "^1.0.1",
+        "object.values": "^1.1.7",
+        "semver": "^6.3.1",
+        "tsconfig-paths": "^3.15.0"
+      },
+      "engines": {
+        "node": ">=4"
+      },
+      "peerDependencies": {
+        "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8"
+      }
+    },
+    "node_modules/eslint-plugin-import/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "dev": true,
+      "dependencies": {
+        "ms": "^2.1.1"
+      }
+    },
+    "node_modules/eslint-plugin-import/node_modules/doctrine": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
+      "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==",
+      "dev": true,
+      "dependencies": {
+        "esutils": "^2.0.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/eslint-plugin-react": {
      "version": "7.34.2",
      "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.34.2.tgz",
@@ -5603,6 +5745,15 @@
        "node": "*"
      }
    },
+    "node_modules/minimist": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
    "node_modules/ms": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -5731,6 +5882,20 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/object.groupby": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/object.groupby/-/object.groupby-1.0.3.tgz",
+      "integrity": "sha512-+Lhy3TQTuzXI5hevh8sBGqbmurHbbIjAi0Z4S63nthVLmLxfbj4T54a4CfZrXIrt9iP4mVAPYMo/v99taj3wjQ==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.7",
+        "define-properties": "^1.2.1",
+        "es-abstract": "^1.23.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
    "node_modules/object.hasown": {
      "version": "1.1.4",
      "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.4.tgz",
@@ -6584,6 +6749,15 @@
        "node": ">=8"
      }
    },
+    "node_modules/strip-bom": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+      "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
    "node_modules/strip-json-comments": {
      "version": "3.1.1",
      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
@@ -6677,6 +6851,30 @@
        "node": ">=4"
      }
    },
+    "node_modules/tsconfig-paths": {
+      "version": "3.15.0",
+      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz",
+      "integrity": "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==",
+      "dev": true,
+      "dependencies": {
+        "@types/json5": "^0.0.29",
+        "json5": "^1.0.2",
+        "minimist": "^1.2.6",
+        "strip-bom": "^3.0.0"
+      }
+    },
+    "node_modules/tsconfig-paths/node_modules/json5": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
+      "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
+      "dev": true,
+      "dependencies": {
+        "minimist": "^1.2.0"
+      },
+      "bin": {
+        "json5": "lib/cli.js"
+      }
+    },
    "node_modules/tslib": {
      "version": "2.6.3",
      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
@@ -34,6 +34,7 @@
    "@types/react-dom": "^18.3.0",
    "@vitejs/plugin-react": "^4.3.1",
    "eslint": "^8.57.0",
+    "eslint-plugin-import": "^2.29.1",
    "eslint-plugin-react": "^7.34.2",
    "eslint-plugin-react-hooks": "^4.6.2",
    "eslint-plugin-react-refresh": "^0.4.7",
@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { useTheme } from '@mui/material/styles';
 import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css';
+import './DataGrid.css'
 import { useMemo, useState } from 'react';

 export default function DataTable({ columns, rows }) {
@@ -23,7 +23,7 @@ export default function DataTable({ columns, rows }) {
        headerName: col.name,
        width: columnWidths[col.name],
      };
-    };
+    }
  }, [columnWidths]);

  return (
@@ -13,7 +13,7 @@ function getGridColDef(col) {
 export default function ReactDataGrid({columns, rows}) {
  return (
    <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  );
+  )
 }

 ReactDataGrid.propTypes = {
@@ -3,7 +3,7 @@ import Button from '@mui/material/Button';
 import { useTheme } from '@mui/material/styles';
 import PropTypes from 'prop-types';
 import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded";
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
 import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
 import FileCopyIcon from '@mui/icons-material/FileCopy';
 import './SideBar.css';
@@ -51,7 +51,7 @@ const Sidebar = ({ onSelection, onOpened, onClosed }) => {
      } catch (error) {
        console.error('Error fetching WebSocket config:', error);
      }
-    };
+    }
    fetchCannedQueries();
  }, []);

@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
+import { defineConfig, loadEnv } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -29,13 +29,12 @@ if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
  }
 }

-const target = env.ASPNETCORE_HTTPS_PORT
-  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
-  : env.ASPNETCORE_URLS
-    ? env.ASPNETCORE_URLS.split(';')[0]
-    : 'https://localhost:8443';
-
 export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  const target = env.ASPNETCORE_HTTPS_PORT
+    ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+    : env.ASPNETCORE_URLS
+      ? env.ASPNETCORE_URLS.split(';')[0]
+      : 'https://localhost:8443';
  return {
    plugins: [plugin()],
    resolve: {
@@ -52,18 +51,7 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
      proxy: {
        '^/api': {
          target,
-          secure: false,
-          configure: (proxy) => {
-            proxy.on('error', (err) => {
-              console.log('proxy error', err);
-            });
-            proxy.on('proxyReq', (proxyReq, req) => {
-              console.log(`Forwarding ${proxyReq.method} ${proxyReq.protocol}//${proxyReq.host}${proxyReq.path} to ${req.socket.remoteAddress}${req.url}`);
-            });
-            proxy.on('proxyRes', (proxyRes, req, res) => {
-              console.log(`Forwarding response ${req.socket.localAddress} to ${res.socket.remoteAddress}. Status=${proxyRes.statusCode}`);
-            });
-          },
+          secure: false
        },
        '^/ws': {
          target,
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
    <ImplicitUsings>enable</ImplicitUsings>
    <InvariantGlobalization>true</InvariantGlobalization>
@@ -12,10 +12,10 @@

  <ItemGroup>
    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="8.0.18" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="8.0.18" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
  </ItemGroup>

  <ItemGroup>
@@ -1,37 +1,30 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using Bts.DataStreamCore;
-
-using Microsoft.AspNetCore.Mvc;
-
-[Route("/ws")]
-public class DataStreamSubscriptionController : ControllerBase {
-   readonly ILogger logger;
-   readonly IDataStreamFactory streamFactory;
-
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
-      this.logger = logger;
-      logger.LogInformation("Creating {ControllerType}", GetType().Name);
-      var streamFactoryList = streamFactories.ToList();
-      streamFactory = streamFactoryList.Count == 1
-         ? streamFactoryList[0]
-         : new CompositeDataStreamFactory(streamFactoryList);
-   }
-
-   [Route("/ws")]
-   public async Task Get(CancellationToken cancellationToken) {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      logger.LogInformation("Establishing websocket connection...");
-      try {
-         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-         logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
-         await server.Run(cancellationToken);
-      } catch (Exception ex) {
-         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
-      }
-   }
-}
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}
@@ -2,7 +2,6 @@ namespace Bts.DataStreamWebApp.Server.Controllers;

 using bts.websocket_core.json_rpc;

-using Bts.DataStreamCore;
 using Bts.DataStreamWebApp.Server.Messages;

 using System.Net.WebSockets;
@@ -14,11 +13,9 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
   readonly ILogger logger;
   readonly WebSocket webSocket;
   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
-   readonly IDataStreamFactory streamFactory;

-   public DataStreamWebSocketServer(WebSocket webSocket, IDataStreamFactory streamFactory, ILogger logger) {
+   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
      this.webSocket = webSocket;
-      this.streamFactory = streamFactory;
      this.logger = logger;
   }

@@ -70,7 +67,7 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
      }
      var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
      disposeSubscription(subId);
-      if (!subReq.ParseQuery(streamFactory, out var vs, out parseErr, out errCode)) {
+      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
@@ -1,11 +1,7 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
-ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
-
 RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
+COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
+COPY --chown=bts:bts "build/Bts.DataStreamWebApp.Client/production/dist" "bin/wwwroot"
+COPY --chown=bts:bts "src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -4,7 +4,6 @@ using BCG;

 using bts.websocket_core.json_rpc;

-using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;

 using System.Diagnostics.CodeAnalysis;
@@ -38,12 +37,7 @@ public class SubscribeRequest {
      return true;
   }

-   internal bool ParseQuery(
-      IDataStreamFactory streamFactory,
-      [NotNullWhen(true)] out ViewStream? stream,
-      [NotNullWhen(false)] out SubscriptionError? error,
-      out ErrorCode errorCode
-   ) {
+   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
      stream = null;
      errorCode = ErrorCode.None;
      error = null;
@@ -60,7 +54,7 @@ public class SubscribeRequest {
      }
      Exception? exception = null;
      try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
      } catch (Exception ex) {
         exception = ex;
      }
@@ -1,6 +1,5 @@
 using Bts.DataStreamCore.Services;
 using Bts.PubSub.Services;
-using Bts.SharedStreams.Services;

 using Spark.Api.Services;
 using Spark.Core.Services;
@@ -79,9 +78,5 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
      .AddSparkApi(config.GetSection("SparkApi"))
      .AddMasterWorker(config.GetSection("MasterWorker"))
      .AddViewStreamDatabase()
-      .AddPrimitiveStreams(config.GetSection("PrimitiveStreams"))
-      .AddMarketDataStreams()
-      .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
-      .AddCompositeDataStreamFactory()
      .AddPubSubClient(config.GetSection("PubSub"));
 }
@@ -1,4 +1,14 @@
 {
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://localhost:8080"
+      },
+      "Https": {
+        "Url": "https://localhost:8443"
+      }
+    }
+  },
  "PubSub": {
    "Routing": {
      "PublisherRoutes": [
@@ -1,45 +0,0 @@
-{
-  "NaturalLanguagePrompt": {
-    "prompts": [
-      {
-        "category": "Impact of financial events on the market",
-        "description": "Show how trading metrics (e.g., price movement, volume) are affected by events related to finance (rate announcements, earnings reports, etc.).",
-        "prompt": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
-        "examples": [
-          {
-            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
-            "dsl": [
-              "time_idx=EventTimeIndex.OpecMeetings(which: \"last 5\", duration: \"+5min\")",
-              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
-              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
-            ]
-          },
-          {
-            "natural_language": "How do OPEC meetings affect the price of Crude futures? Examine the 10 minutes after each meeting this year.",
-            "dsl": [
-              "time_idx=EventTimeIndex.OpecMeetings(duration: \"+10min\", predicate: \"timestamp.year==2024\")",
-              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
-              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
-            ]
-          },
-          {
-            "natural_language": "Show me how the SP 500 price moves after FOMC rate announcements that day",
-            "dsl": [
-              "time_idx=EventTimeIndex.FomcRateAnnouncements(which: \"timestamp.year==2024\", duration: \"+10min\")",
-              "contract_idx=ActiveFutureContracts(predicate: \"product==ES\")",
-              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
-            ]
-          },
-          {
-            "natural_language": "Analyze the impact of the FED's interest rate hikes in 2022 on the 10-year Treasury yield, focusing on the 2-week period following each hike.",
-            "dsl": [
-              "time_idx=FedInterestRateAnnouncements(predicate: \"timestamp.year==2022 && change_pct>0\", duration: \"+2 weeks\")",
-              "contract_idx=ActiveFutureContracts(predicate: \"product==ZN\")",
-              "Bars(time_index: time_idx, contract_index: contract_idx)"
-            ]
-          }
-        ]
-      }
-    ]
-  }
-}
@@ -30,15 +30,5 @@
        "benzinga_news"
      ]
    }
-  ],
-  "Spark": {
-    "ConfigServer": "http://sparkd:8000/config"
-  },
-  "SparkApi": {
-    "IsEnabled": true,
-    "Credentials": {
-      "User": "bts",
-      "Password": ""
-    }
-  }
+  ]
 }
@@ -11,14 +11,8 @@
      "Http": {
        "Url": "http://0.0.0.0:8080"
      },
-      "HttpIpv6": {
-        "Url": "http://[::1]:8080"
-      },
      "Https": {
        "Url": "https://0.0.0.0:8443"
-      },
-      "HttpsIpv6": {
-        "Url": "https://[::1]:8443"
      }
    }
  },
@@ -28,358 +22,6 @@
  "ViewStreamDatabase": {
    "Interval": "00:00:00.1"
  },
-  "PrimitiveStream": {
-    "Assemblies": [
-      "DataStream.dll",
-      "SharedStreams.dll"
-    ],
-    "ExcludedTypes": [
-      "SettlementPriceStream",
-      "OHLCVStream"
-    ]
-  },
-  "StreamDatabase": {
-    "Path": "/home/bts/.local/share/bts/silver_pine/store.db",
-    "Query": "select * from metadata"
-  },
-  "NaturalLanguageStream": {
-    "prompts": [
-      {
-        "category": "Settlement",
-        "MulitLineDescription": [
-          "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). ",
-          "The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). ",
-          "The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement ",
-          "(symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have ",
-          "datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know ",
-          "what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however ",
-          "you should use the Today's real date"
-        ],
-        "prompt": "What was the settlement for SP 500 Future on June 8th?",
-        "examples": [
-          {
-            "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
-            "dsl": "Settlement(\"ES\", 4/22/2023)"
-          },
-          {
-            "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.",
-            "dsl": "Settlement(\"SIZ4\", 4/22/2023)"
-          },
-          {
-            "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.",
-            "dsl": "Settlement(\"CL\", 10/10/2024)"
-          },
-          {
-            "natural_language": "Find today's settlement of the 10 year treasury future.",
-            "dsl": "Settlement(\"ZN\", 10/11/2024)"
-          },
-          {
-            "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.",
-            "dsl": "Settlement(\"NGZ4\", 9/6/2024)"
-          },
-          {
-            "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.",
-            "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)"
-          },
-          {
-            "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25  spread for the first two weeks of September.",
-            "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)"
-          },
-          {
-            "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.",
-            "dsl": "Settlement(\"NG\", 9/6/2024)"
-          }
-        ]
-      },
-      {
-        "category": "BBO",
-        "MulitLineDescription": [
-          "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ",
-          "It accepts the following parameters: ",
-          "symbol (a product code such as ZN for 10 year T-Note futures), ",
-          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
-          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
-          "interval (sample frequency of the data), ",
-          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
-          "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
-          "Depending on the wording of the request, some arguments may be omitted. ",
-          "The begin, end, and interval args should be expressed in iso 8601 format. ",
-          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
-          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
-          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
-        ],
-        "prompt": "Show me bbo for SP 500 futures on Sep 23rd of this year.",
-        "examples": [
-          {
-            "natural_language": "Get SP500 future bbo from Wednesday.",
-            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")"
-          },
-          {
-            "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
-            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", exclusive_end: true)"
-          },
-          {
-            "natural_language": "Retrieve NVIDIA L1 books from Oct 3",
-            "dsl": "Bbo(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\")"
-          },
-          {
-            "natural_language": "Show me L1 data for Crude oil futures this week",
-            "dsl": "Bbo(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\")"
-          },
-          {
-            "natural_language": "Fetch hourly bbo for Gold futures this week",
-            "dsl": "Bbo(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
-          },
-          {
-            "natural_language": "What was the minute by minute bbo for Tesla yesterday?",
-            "dsl": "Bbo(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
-          }
-        ]
-      },
-      {
-        "category": "Order flow imbalance",
-        "MulitLineDescription": [
-          "This is a few shot learning example for an NL to DSL. The DSL function in question is OrderImbalance(args...). ",
-          "It accepts the following parameters: ",
-          "symbol (a product code such as ZN for 10 year T-Note futures), ",
-          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
-          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
-          "interval (sample frequency of the data), ",
-          "levels (the number of price levels to examine)",
-          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
-          "inclusive_end (a bool flag that, when true, makes the end time inclusive). ",
-          "Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
-          "Depending on the wording of the request, some arguments may be omitted. ",
-          "The begin, end, and interval args should be expressed in iso 8601 format. ",
-          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
-          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
-          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
-        ],
-        "examples": [
-          {
-            "natural_language": "Get SP500 order imbalance from Wednesday.",
-            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", \"PT1M\")"
-          },
-          {
-            "natural_language": "Get SP500 order imbalance from Wednesday for the top 5 price levels.",
-            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", levels: 5, interval: \"PT1M\")"
-          },
-          {
-            "natural_language": "Get orderflow imbalance for 10 year T-Notes in the first minute after the market opened yesterday.",
-            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", interval: \"PT1S\")"
-          },
-          {
-            "natural_language": "Retrieve NVIDIA order flow from Oct 3",
-            "dsl": "OrderImbalance(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\", interval: \"PT1S\")"
-          },
-          {
-            "natural_language": "Show me order imbalance for Crude oil futures this week",
-            "dsl": "OrderImbalance(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
-          },
-          {
-            "natural_language": "Fetch hourly orderflow for Gold futures this week",
-            "dsl": "OrderImbalance(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
-          },
-          {
-            "natural_language": "What was the minute by minute orderflow for Tesla yesterday?",
-            "dsl": "OrderImbalance(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
-          }
-        ]
-      },
-      {
-        "category": "VWAP",
-        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
-        "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
-        "examples": [
-          {
-            "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
-            "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
-          },
-          {
-            "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
-            "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
-          },
-          {
-            "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
-            "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
-          },
-          {
-            "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
-            "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
-          }
-        ]
-      },
-      {
-        "category": "Price Diff",
-        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
-        "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
-        "examples": [
-          {
-            "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
-            "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
-          },
-          {
-            "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
-            "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
-          },
-          {
-            "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
-            "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
-          }
-        ]
-      },
-      {
-        "category": "Absolute Volume",
-        "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
-        "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
-        "examples": [
-          {
-            "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
-            "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
-          },
-          {
-            "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
-            "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
-          },
-          {
-            "natural_language": "Which 10 year Spread traded the highest volume last week?",
-            "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
-          },
-          {
-            "natural_language": "Which Corn Spread traded the most today?",
-            "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
-          },
-          {
-            "natural_language": "Which Gold Future traded the most today?",
-            "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
-          }
-        ]
-      },
-      {
-        "category": "Roll",
-        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ",
-        "prompt": "When does the roll typically occur In  SP 500?",
-        "examples": [
-          {
-            "natural_language": "When does the roll typically occur In CL?",
-            "dsl": "FuturesRoll(\"CL\")"
-          },
-          {
-            "natural_language": "When does the roll typically occur for 10 year treasury?",
-            "dsl": "FuturesRoll(\"ZN\")"
-          },
-          {
-            "natural_language": "When does the roll usually occur for Gold Future",
-            "dsl": "FuturesRoll(\"GC\")"
-          },
-          {
-            "natural_language": "At what date does the future rollover happens for SP500?",
-            "dsl": "FuturesRoll(\"ES\")"
-          },
-          {
-            "natural_language": "When does the future rollover happens for the Corn Future?",
-            "dsl": "FuturesRoll(\"ZC\")"
-          }
-        ]
-      },
-      {
-        "category": "Daily Move",
-        "description": "This is a few shot learning example for an NL to DSL. ",
-        "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?",
-        "examples": [
-          {
-            "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close",
-            "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)"
-          },
-          {
-            "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close",
-            "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)"
-          },
-          {
-            "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open",
-            "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
-          },
-          {
-            "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close",
-            "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
-          }
-        ]
-      },
-      {
-        "category": "Average Volume",
-        "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
-        "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
-        "examples": [
-          {
-            "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ",
-            "dsl": "FuturesRoll(\"ES\")"
-          },
-          {
-            "natural_language": "Usually when does the open interest move from one contract to another in WTI?",
-            "dsl": "FuturesRoll(\"CL\")"
-          }
-        ]
-      },
-      {
-        "category": "Events",
-        "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.",
-        "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?",
-        "examples": [
-          {
-            "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?",
-            "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)"
-          },
-          {
-            "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?",
-            "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)"
-          },
-          {
-            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.",
-            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)"
-          },
-          {
-            "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.",
-            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)"
-          }
-        ]
-      },
-      {
-        "category": "Indicator, Standard Deviation",
-        "MulitLineDescription": [
-          "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when ",
-          "the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, ",
-          "where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and ",
-          "amounts of the last 4+ standard deviation moves in WTI futures\" would be: ",
-          "r=standardDeviation(\"ES\", \"90d\")\n",
-          "r=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\n",
-          "r=filter(r,\"abs_daily_change >= comp\"). ",
-          "The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, ",
-          "and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, ",
-          "abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters ",
-          "(like a sql where) of rows where abs_daily_change >= comp"
-        ],
-        "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP  futures ?",
-        "examples": [
-          {
-            "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?",
-            "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
-          },
-          {
-            "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation",
-            "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
-          },
-          {
-            "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation",
-            "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")"
-          },
-          {
-            "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation",
-            "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
-          }
-        ]
-      }
-    ]
-  },
  "SparkApi": {
    "IsEnabled": false,
    "Credentials": {
@@ -435,7 +77,7 @@
        {
          "TagType": ".*",
          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
+         "Class": "bts.PubsubServerGenericDataHandler"
        }
      ]
    }
@@ -0,0 +1,6 @@
+FROM bluetradingsystems/linux_dotnet7
+
+RUN dotnet dev-certs https --trust
+COPY --chown=bts:bts "bin" "bin"
+COPY --chown=bts:bts "webapp.spark.conf" ".local/share/DataStream/conf/spark.conf"
+WORKDIR "/home/bts/bin"
@@ -1,13 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.7" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
  </ItemGroup>

 </Project>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
-      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
-         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,46 +0,0 @@
-using System;
-
-namespace BCG {
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
-      private int myId = 0;
-      private ViewStream myVS;
-
-      public CountStream(ViewStream vs) {
-         myVS = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += myHandleChanges;
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= myHandleChanges;
-      }
-
-
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
-            myLastFiredDataPacket =
-               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
-                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  myLastFiredDataPacket.newRows)
-            );
-      }
-
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
-         return retVal;
-      }
-
-
-   }
-}
@@ -1,38 +0,0 @@
-namespace bts.BlueOrb {
-   using bts;
-
-   using BCG;
-
-   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,157 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Text.RegularExpressions;
-
-   [DataStreamUserRespresentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      public class RegExpFilter : IStreamFilter {
-         private Regex[] myRegExps;
-         private string[] myCols;
-
-         public RegExpFilter(string col, string regExp)
-            : this(new string[] { col }, new string[] { regExp }) { }
-
-         public RegExpFilter(string[] cols, string[] regExps) {
-            myCols = cols;
-            myRegExps = new Regex[regExps.Length];
-            for (int i = 0; i < regExps.Length; i++)
-               myRegExps[i] = new Regex(regExps[i]);
-         }
-
-         #region IStreamFilter Members
-         public bool filter(IBlueRow dr) {
-            bool rv = true;
-            for (int i = 0; i < myCols.Length; i++) {
-               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-               if (!rv)
-                  return rv;
-            }
-            return rv;
-         }
-         #endregion
-      }
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
-
-      #region constructors
-      public delegate bool OnFilter(IBlueRow r);
-      private class OnFilterWrapper : IStreamFilter {
-         public OnFilterWrapper(OnFilter f) { this.f = f; }
-
-         public bool filter(IBlueRow dr) { return f(dr); }
-
-         private OnFilter f;
-      }
-      public FilterStream(ViewStream vs, OnFilter f)
-         : this(vs, new OnFilterWrapper(f)) { }
-
-
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
-         }
-      }
-
-      public FilterStream(ViewStream vs, string filterExpression)
-         : this(vs, new BooleanExpressionFilter(filterExpression)) {
-         myFilterString = filterExpression;
-      }
-
-      public FilterStream(ViewStream vs, string col, string regExp)
-         : this(vs, new RegExpFilter(col, regExp)) {
-         myFilterString = col + " , " + regExp;
-      }
-
-      public FilterStream(ViewStream vs)
-         : this(vs, new BinaryFilter(true)) {
-         myFilterString = true.ToString();
-      }
-      #endregion
-
-      #region overridden methods
-      protected override void myOnFirstSubscriber() =>
-         myParentStream.onChange += myHandleCallbacks;
-
-      protected override void myOnLastSubscriber() {
-         myLastFiredRows = null;
-         myParentStream.onChange -= myHandleCallbacks;
-      }
-
-      protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
-         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
-            base.myFireEvent(o);
-      }
-      #endregion
-
-      private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
-         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
-      }
-
-      public IStreamFilter filter {
-         set {
-            {
-               myFilter = value;
-               if (numberOfSubscribers > 0) {
-                  if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
-                  }
-               }
-            }
-         }
-         get {            
-               return myFilter;
-         }
-      }
-
-      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
-
-      public override string ToString() {
-         return GetType().ToString() + " " + myFilterString;
-      }
-
-      private void myHandleFilterChange() {
-         filter = myFilter;
-      }
-   }
-}
@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("GroupByStream")]
-   public class GroupByStream : ViewStream {
-
-      public interface INonIncGroupByEvaluator {
-         // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
-      }
-
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
-         _main_stream = main_stream;
-         _groupby_func = groupby_func;
-         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
-      }
-
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
-
-      private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
-         ///grouping together
-         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
-         /// reducing
-         _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
-            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}
@@ -1,187 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using RowHash = System.Int32;
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   using System.Collections;
-   using System.Data.Entity.Core.Mapping;
-
-   [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
-         // initialized in cons
-         private readonly List<string> _join_col_names;
-         private readonly bool _is_left;
-         private readonly BlueRowHasher _hasher;
-         // initialized on first udpate         
-         private IBlueTableSchema _schema;
-         private int[] _join_cols;
-         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-         
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
-            _is_left = is_left;
-            _hasher = row_hasher;
-         }
-
-         public IBlueTableSchema schema => _schema;
-
-         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
-
-         public List<IBlueRow> get_matching_rows (int hash) {
-            List<IBlueRow> rv = null;
-            _hash_row_map.TryGetValue(hash, out rv);
-            return rv;
-         }
-
-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) { 
-               return 0;
-            }
-            return _hasher.get_hash(r, _join_cols);
-         }
-
-         public void update_data(DataPacket dp) {
-            if (!ready) {               
-               _schema = dp.newRows.schema;
-               _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
-            }
-            _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
-               int hash = _get_hash(r);
-               List<IBlueRow> hash_rows = null;
-               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
-                  hash_rows = new List<IBlueRow>();
-                  _hash_row_map.Add(hash, hash_rows);
-               }
-               hash_rows.Add(r);
-            }
-         }
-
-         public bool ready {
-            get {
-               return _schema != null;
-            }
-         }
-
-      }
-
-      private ViewStream _ls, _rs;
-      private JoinBookkeepingData _ldata, _rdata;
-      private ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema = null;
-
-      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
-
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
-         _ls = ls;
-         _rs = rs;
-         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count ==  rj_col_list.Count);
-         int n_cols = lj_col_list.Count;
-         BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _ls.onChange += _ls_onchange;
-         _rs.onChange += _rs_onchange;
-      }
-
-      private void _ls_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         _ldata.update_data(dp);
-         _form_and_fire();
-      }
-
-      private void _rs_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket ;
-         _rdata.update_data(dp);
-         _form_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         _ls.onChange -= _ls_onchange;
-         _rs.onChange -= _rs_onchange;
-      }
-
-      private void _build_schema() {
-         if (_schema != null)
-            return;
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
-         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
-            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
-            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
-      }
-
-      private void _form_and_fire() {
-         if (!_ldata.ready||!_rdata.ready)
-            return;
-         _build_schema();
-         List<object[]> res_rows = new List<object[]>();
-         
-         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.schema.columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         };
-         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
-            object[] rv = new object[_schema.columnNames.Length];            
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         };
-         // this is where the join happens
-         foreach (var kv in _ldata.rows) {
-            RowHash hash = kv.Key;
-            List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
-
-            foreach (IBlueRow lr in l_rows) {                              
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
-                     object[] rowvals = build_row_array(res_rows.Count, lr);
-                     copy_row(rowvals, r, false);
-                     res_rows.Add(rowvals);
-                  }
-               } else {
-                  object[] rowvals = build_row_array(res_rows.Count, lr);
-                  res_rows.Add(rowvals);
-               }
-            }
-         }
-         BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
-            rv.addRow(new BlueRow(_schema, rval));
-         }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-   }
-}
@@ -1,28 +0,0 @@
-//namespace bts.Blueorb {
-//   using BCG;
-
-//   public class JsonSaverStream : ViewStream {
-//      private ViewStream _stream;
-//      private string _file_name;
-//      public JsonSaverStream(ViewStream vs, string file_name) {
-//         _stream = vs;
-//         _file_name = file_name;
-//      }
-
-//      protected override void myOnFirstSubscriber() {
-//         _stream.onChange += _on_data;
-//      }
-
-//      private void _on_data(object dp_obj) {
-//         if (dp_obj == null) {
-//            return;
-//         }
-//         DataPacket dp = dp_obj as DataPacket;
-//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
-//      }
-
-//      protected override void myOnLastSubscriber() {
-//         _stream.onChange -= _on_data;
-//      }
-//   }
-//}
@@ -1,91 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-using Amazon.Runtime.Internal.Util;
-
-using BCG;
-using BCG.Utilities;
-
-using bts;
-
-namespace bts.BlueOrb{
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      // col_name: col_value ... ; col_name: col_value
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
-
-
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
-         int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
-            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
-            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
-            n++;
-         }
-
-
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
-         n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
-                  continue;
-               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
-               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
-            }
-
-         mySetRealSource(new BlueRowCollectionStaticStream(t));
-      }
-   }
-}
@@ -1,89 +0,0 @@
-using System;
-
-using BCG.Utilities.General;
-using BCG;
-using bts;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("MillisecondStream")]
-   public class MillisecondStream : ViewStream, IWorker {
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private readonly TimeSpan _period;
-      private readonly int _size;
-      private readonly ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema;
-      private bts.CircularBufferTable _table;
-
-      public MillisecondStream(int interval_msecs, int size)
-      : this(ViewStreamDatabase.instance, interval_msecs, size) {
-      }
-
-      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-         _vsd = vsd;
-         _period = TimeSpan.FromMilliseconds(interval_msecs);
-         _size = size;
-         _schema = BlueDataUtility.form_schema("Milliseconds",
-               Tuple.Create("id", TypeCode.Int32),
-               Tuple.Create("time", TypeCode.DateTime),
-               Tuple.Create("display_time", TypeCode.String));
-         _table = new CircularBufferTable(_schema, _size);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private int _id = 0;
-      public void doWork() {
-         if (DateTime.Now - _last_fire_time >= _period) {
-            _last_fire_time = DateTime.Now;
-            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
-            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
-   }
-   internal class MSecondStream : TimeUnitStream {
-      public MSecondStream(int intervalInms, int size)
-         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
-
-      protected override DateTime myRoundTime(DateTime dt) { return dt; }
-   }
-
-   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
-      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
-                                                               new ColumnInfo("id", 0, typeof(int)),
-                                                               new ColumnInfo("time", 1, typeof(string)));
-      public DisplayableMillisecondStream(int intervalInMs, int sz) {
-         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
-      }
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         DateTime time = row.get_datetime("time");
-         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-         return new BlueRow(mySchema, row["id"], time_txt);
-      }
-
-      #endregion
-
-      private string myToString(DateTime dt) {
-         string sep = ":";
-         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
-      }
-   }
-}
@@ -1,161 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-   }
-   public class MultiRowOpHelper {
-      // input + op
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse_ops(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rows.ElementAt(start_idx);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         // out_col vs inp_cal + type
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-            or.setValue("id", ir["id"]);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.Median:
-               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
-            //case MultiRowOp.Mode:
-            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}
@@ -1,40 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System.Collections.Generic;
-   using System.Net;
-
-   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
-   public class MultiRowRollingMathStream : ViewStream {
-      private ViewStream _sorted_parent_str;
-      private MultiRowOpHelper _op;
-      private int _rolling_window_sz;
-      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
-         _sorted_parent_str = new SortStream(parent_str, sort_exp);
-         _op = MultiRowOpHelper.parse_ops(op_txt);
-         _rolling_window_sz = rolling_window_sz;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _sorted_parent_str.onChange += _parent_str_onchange;
-      }      
-
-      protected override void myOnLastSubscriber() {
-         _sorted_parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         var rows = dp.newRows.to_list();
-         BlueTable out_table = null;
-         for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
-            if (out_table == null)
-               out_table = new BlueTable(out_r.schema);
-            out_table.addRow(out_r);
-         }            
-         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -1,188 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   #region mappings
-   internal class ProjectData {
-      // schema from the base stream
-      public readonly IBlueTableSchema in_schema;
-      // this is the projected schema
-      public readonly IBlueTableSchema out_schema;
-      // projected cols to the input cols, for cols who's names are changed
-      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
-      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
-                  IDictionary<string, string> mapped_to_orig_col_map_) {
-         in_schema = in_schema_;
-         out_schema = out_schema_;
-         mapped_to_orig_col_map = mapped_to_orig_col_map_;
-      }
-   }
-   #endregion
-
-
-#region Row with mappings build in
-   internal class NewProjectRow : IBlueRow {
-      #region IBlueRow members
-      private ProjectData _project_data;
-      private IBlueRow _row;
-      public NewProjectRow(ProjectData project_data, IBlueRow row) {
-         _project_data = project_data;
-         _row = row;
-      }
-      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
-      #endregion
-
-      #region ICollection members
-      public int Count { get { return schema.columnNames.Length; } }
-
-      public bool IsSynchronized { get { return false; } }
-
-      public object SyncRoot { get { return this; } }
-
-      public void CopyTo(Array array, int index) {
-         int i = index;
-         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
-      }
-      #endregion
-
-      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
-
-      public bool IsReadOnly { get { return true; } }
-
-      public object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            return this[(int)key];
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public object this[int index] {
-         get {
-            return this[schema.columnInfo[index].colName];
-         }
-      }
-
-      public object this[string col_name] {
-         get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
-               return _row[mapped_to_col];
-            }
-            return _row[col_name];
-         }
-      }
-
-      public ICollection Keys { get { return schema.columnNames; } }
-
-      public ICollection Values {
-         get {
-            object[] retVal = new object[schema.columnNames.Length];
-            CopyTo(retVal, 0);
-            return retVal;
-         }
-      }
-
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public bool Contains(object key) { return schema[(string)key] != null; }
-
-      /// <summary>
-      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
-      /// iterate through the values, typecast this into an IEnumerable
-      /// </summary>
-      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-      #endregion
-   }
-   #endregion
-
-   [DataStreamUserRespresentation("ProjectStream")]
-   public class NewProjectStream : ViewStream, IStreamMapper {
-
-      private ViewStream _stream;
-      private string _project_expression;
-      private ProjectData _project_data = null;
-      //public ProjectData 
-      public NewProjectStream(ViewStream vs, string project_expression) {
-         _stream = vs;
-         _project_expression = project_expression;
-         _stream = new MapStream(vs, this);
-         mySetRealSource(_stream);
-      }
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-         _project_data = _create_project_data(input_schema, _project_expression);
-         return _project_data.out_schema;
-      }
-
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
-
-      // this is where most of the logic is, in terms of translating cols
-      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
-                                                      string project_expression) {
-         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ', ',');
-         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // modifications (6/30/2025):
-         // only the columns represented here, either mapped
-         // or not, will show
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa b") -> aa b
-         foreach (string o_p_tok in project_toks) {
-            string p_tok = o_p_tok.Trim();
-            if (p_tok.Contains(":")) {
-               // explicit mapping
-               string[] map_toks = p_tok.Split(":");
-               string orig_col = map_toks[0];
-               string mapped_col = map_toks[1];
-               orig_to_mapped.Add(orig_col, mapped_col);  
-            } else {
-               // this is a col_name explicitely stated
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
-            }
-         }
-
-         // make the new schema
-         List<ColumnInfo> new_cols = new List<ColumnInfo>();
-         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;           
-            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
-               string mapped_name = orig_to_mapped[ci.colName];
-               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-            }
-         }
-         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
-         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
-      }
-
-   
-
-   }
-}
@@ -1,127 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System.IO;
-   using System.Diagnostics;
-   using System;
-
-   [DataStreamUserRespresentation("PythonDataframeStream")]
-   public class PythonDataframeStream : ViewStream {
-      private ViewStream _parent_str;
-      private readonly string _in_csv_file_name;
-      private readonly string _out_csv_filename;
-      private readonly string _transform_function;
-      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
-         var guid = System.Guid.NewGuid();
-         _in_csv_file_name = $"pdfs_in_{guid}.csv";
-         _out_csv_filename = $"pdfs_out_{guid}.csv";
-         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
-         _transform_function = transform_function;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _parent_str_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         Log.assert(dp_obj != null);
-         // the file _in_csv_file_name is now populated.
-         // call py_func_wrapper_main and pass this file name as well
-         // as the output file name
-         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
-         // transform function, which returns the tranformed df. py_func_wrapper_main then
-         // writes it into the _out_csv_filename file.
-         // now this function should read _out_csv_filename into a dp
-         // and fire it
-         _compute_and_write();
-         ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
-         IBlueRowCollection nrows = null;
-         rs.onChange += (dp_obj) => {
-            nrows = (dp_obj as DataPacket).newRows;
-         };
-         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      private void _compute_and_write() {
-         File.WriteAllText("user_code.py", _transform_function);
-         // wrapper function
-         string py_func_wrapper_main = @"
-import sys
-from user_code import transform
-import pandas as pd
-
-if __name__ == '__main__':
-   input_file_name = sys.argv[1]
-   output_file_name = sys.argv[2]
-# 1. load input_file_name into df
-# 2. do some test calc
-# 3. write the output of 2 into output_file_name
-   df = pd.read_csv(input_file_name)   
-   df_out = transform(df)
-   df_out.to_csv(output_file_name, index=False)";
-
-
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-      }
-
-      #region Unused test code
-      private void _simple_test() {
-
-         // actual function that needs to be called ////////////////////////
-         string py_real_function = @"
-def test(x, out_path):
-   rv = x * 10
-   with open(out_path, 'w') as f:
-      f.write(str(rv))
-   return rv";
-         // the function is in a string object.
-         File.WriteAllText("user_code.py", py_real_function);
-
-
-         // runner app (top level main function) //////////////////////////////
-         string py_func_wrapper_main = @"
-import sys
-from user_code import test
-
-if __name__ == '__main__':
-   arg = int(sys.argv[1])
-   out_path = sys.argv[2]
-   rv = test(arg, out_path)
-   rv = rv + 12
-   with open(out_path, 'w') as f:to \
-      f.write(str(rv))";
-   
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         string outputPath = "py_out.txt";
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py 4 {outputPath}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-         // Now read the output written by the function
-         string result = File.ReadAllText(outputPath);
-         Console.WriteLine(result); // Should print "6"
-
-      }
-      #endregion
-   }
-}
@@ -1,30 +0,0 @@
-using BCG;
-
-namespace bts.BlueOrb {
-   public class SchemaGuesserStream : ViewStream {
-
-      private IBlueRowCollection _rows = null;
-      private ViewStream _vs;
-      public SchemaGuesserStream(ViewStream vs) {
-         _vs = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vs.onChange += _on_data;
-      }
-      protected override void myOnLastSubscriber() {
-         _vs.onChange -= _on_data;
-      }
-
-      private void _on_data(object obj) {
-         DataPacket dp = obj as DataPacket;
-         if (_rows == null) {
-            _rows = BlueDataUtility.get_typed_table(dp.newRows);
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-
-   }
-}
@@ -1,67 +0,0 @@
-using Bts.ArithmeticExpressions;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG;
-
-namespace bts.BlueOrb {
-
-   [DataStreamUserRespresentation("SingleRowMathStream")]
-   public class MathStream : ViewStream, IStreamMapper {
-      private IArithmeticProgram myExp;
-      private BooleanExpressionFilter _bool;
-      List<string> myExtraColumns = new List<string>();
-      BlueTableSchema mySchema;
-
-
-      public MathStream(ViewStream vs, string mathExpression)
-         : this(vs, mathExpression, string.Empty) {
-      }
-
-      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
-         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
-            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
-         myExtraColumns = myExp.OutputIdentifiers.ToList();
-         if (!string.IsNullOrEmpty(boolean_expression))
-            _bool = new BooleanExpressionFilter(boolean_expression);
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
-            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
-            foreach (ColumnInfo ci in inputSchema.columnInfo)
-               colIdx[ci.colName] = ci;
-
-            foreach (string ncn in myExtraColumns) {
-               ColumnInfo newC;
-               if (colIdx.ContainsKey(ncn))
-                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
-               else
-                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
-               colIdx[ncn] = newC;
-            }
-            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         MutableBlueRow result = new MutableBlueRow(mySchema, row);
-         if (_bool == null || _bool.filter(row)) {
-            myExp.Evaluate(result.AsDictionary());
-         }
-         return result;
-      }
-
-      #endregion
-   }
-}
@@ -1,17 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SortStream")]
-   public class SortStream: ViewStream {
-      public SortStream (ViewStream parent_stream, string sort_expr) {
-         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
-   public class TimeBasedRollingFilterStream : ViewStream {
-      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
-         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("TopStream")]
-   public class TopStream : ViewStream { 
-      public TopStream(ViewStream vs, int n_rows, string sort_col) {
-         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-   using System.Collections.Generic;
-
-   using mbts;
-   using BCG;
-
-   [DataStreamUserRespresentation("UnionStream")]
-   public class UnionStream : ViewStream {
-      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
-         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -18,13 +18,8 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="IronPython" Version="3.4.2" />
-    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
  </ItemGroup>

 </Project>
@@ -2,26 +2,15 @@ namespace Bts.DataStream;

 using BCG;

-using bts.api.core;
-
-using Bts.DataStreamCore;
-
-using Microsoft.Extensions.Logging;
-
 using Spark.Core.Disposables;

 using System;
-using System.Collections;
-using System.Linq;
 using System.Reactive.Linq;

 public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
      source as ViewStream ?? new ObservableViewStream(source);

-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
-
   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);

@@ -34,119 +23,4 @@ public static class DataStreamExtensions {
               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
      }
   }
-
-   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-      source.Select(dp => {
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
-
-         IBlueRowCollection<T> CastRows(ICollection rows) =>
-            new BlueTableSelectorWrapper<T>(
-               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-      });
-
-   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
-      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
-      source.AsTables<T, int>(valueSelector, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
-      source.AsTables<T, int>(valueSelector, keySelector);
-
-   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
-      rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
-         var bd = res.to_builder();
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         foreach (IBlueRow r in addedRows) {
-            bd.add_or_update(valueSelector(r));
-         }
-         foreach (IBlueRow r in deletedRows) {
-            bd.remove(rowKeySelector(r));
-         }
-         return bd.to_table();
-      });
-   }
-
-   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.Scan(
-         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
-         (res, table) => {
-            var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.added) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
-         .Select(t => t.Dp);
-
-   public static IObservable<IDataPacket<BlueRowAdapter<T>>> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
-      return Observable.Create<IDataPacket<BlueRowAdapter<T>>>(Subscribe);
-
-      IDisposable Subscribe(IObserver<IDataPacket<BlueRowAdapter<T>>> o) {
-         var adapter = new BlueTableImmutableAdapter<T>(schema);
-         IDataPacket<BlueRowAdapter<T>> initDataPacket = DataPacket<BlueRowAdapter<T>>.Empty(schema);
-         return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
-
-         (BlueTableImmutableAdapter<T> Adapter, IDataPacket<BlueRowAdapter<T>> DataPacket) OnUpdate(
-            (BlueTableImmutableAdapter<T> Adapter, IDataPacket<BlueRowAdapter<T>> DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
-            var bd = state.Adapter.ToDataPacketBuilder();
-            bd.Update(tableUpdate);
-            var dp = bd.ToDataPacket();
-            var tableAdapter = bd.ToImmutable();
-            return (bd.ToImmutable(), dp);
-         }
-      }
-   }
-
-   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.AsDataPackets(schema).AsViewStream();
-
-   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
-      var indices = ResolveIndices(schema, new[] { name });
-      return indices[0];
-   }
-
-   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
-      var indices = ResolveIndices(schema, new[] { name1, name2 });
-      return (indices[0], indices[1]);
-   }
-
-   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
-      return (indices[0], indices[1], indices[2]);
-   }
-
-   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
-      return (indices[0], indices[1], indices[2], indices[3]);
-   }
-
-   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
-      var columns = schema.columnInfo;
-      var indices = new int[columnNames.Length];
-      int nResolved = 0;
-      for (int i = 0; i < columnNames.Length; i++) {
-         int origNResolved = nResolved;
-         for (int j = 0; j < columns.Length; j++) {
-            if (columns[j].colName == columnNames[i]) {
-               indices[i] = j;
-               nResolved++;
-               break;
-            }
-         }
-         if (nResolved == origNResolved) {
-            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
-         }
-      }
-      return indices;
-   }
 }
@@ -1,32 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using BCG;
-   using System.Collections.Generic;
-   using Bts.DataStreamExpressions;
-   using Bts.DataStreamCore;
-
-   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
-   // So for example, if an expresison is passed 
-   [DataStreamUserRespresentation("dsl")]
-   public class DSLStream : ViewStream {
-      private readonly string _dsl;
-      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
-         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
-         _dsl = dsl;
-         var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
-         mySetRealSource(rs);
-      }
-
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
-         try {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
-            return rv;
-
-         } catch (Exception ex) {
-            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-   }
-}
@@ -4,21 +4,14 @@ using BCG;

 using bts;

-using Microsoft.Extensions.Logging;
-
 using System;

-public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDisposable {
-   readonly IObservable<IDataPacket> source;
-   readonly string? name;
-   readonly ILogger? logger;
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
   IDisposable? subscription;

-   public ObservableViewStream(IObservable<IDataPacket> source, string? name = null, ILogger? logger = null) {
+   public ObservableViewStream(IObservable<DataPacket> source) =>
      this.source = source;
-      this.name = name;
-      this.logger = logger;
-   }

   public void Dispose() {
      GC.SuppressFinalize(this);
@@ -30,13 +23,7 @@ public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDispo

   protected override void myOnFirstSubscriber() {
      Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
-
-      void OnError(Exception exception) =>
-         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
-
-      void OnCompleted() =>
-         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
+      subscription = source.Subscribe(myFireEvent);
   }

   protected override void myOnLastSubscriber() {
@@ -45,6 +32,6 @@ public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDispo
      subscription = null;
   }

-   IDisposable IObservable<IDataPacket>.Subscribe(IObserver<IDataPacket> observer) =>
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
      source.Subscribe(observer);
 }
@@ -11,7 +11,6 @@ using Spark.Core.Disposables;

 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Reactive.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -35,12 +34,12 @@ namespace BCG {
            view_publisher = view_publisher_;
         }
      }
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
+      static ViewStream? build_stream(string stream_expression)
      {
         try
         {
            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
            return rv;

         } catch (Exception ex) {
@@ -52,20 +51,15 @@ namespace BCG {
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
      readonly IPubTableBroker _broker;
      readonly IManagedSubscriber _sub;
-      readonly IDataStreamFactory streamFactory;
      readonly object syncRoot = new();
      IDisposable? cloudStreamsSub;
      IDisposable? remoteRequestsSub;
      bool isDisposed;


-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IEnumerable<IDataStreamFactory> streamFactories) {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
         _sub = managedSubscriber;
         _broker = pubTableBroker;
-         var streamFactoryList = streamFactories.ToList();
-         streamFactory = streamFactoryList.Count == 1
-            ? streamFactoryList[0]
-            : new CompositeDataStreamFactory(streamFactoryList);
      }

      public Task StartAsync(CancellationToken cancellationToken) {
@@ -73,9 +67,7 @@ namespace BCG {
            Log.assert(!isDisposed);
            Log.isnull(cloudStreamsSub);
            Log.isnull(remoteRequestsSub);
-            var cloudStream = streamFactory.Create("table", new (string?, int?, object?)[] { ("name", 0, "cloud_streams") })
-               ?? throw new Exception("Failed to create TableStream 'cloud_stream'");
-            cloudStreamsSub = cloudStream.AsObservable().Subscribe(_on_str);
+            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
@@ -110,11 +102,7 @@ namespace BCG {

      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         var stream = build_stream(streamFactory, stream_expression);
-         if (stream is null) {
-            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
-            return;
-         }
+         ViewStream stream = build_stream(stream_expression);
         _expose_to_ws(name, stream);
         if (persist) {
            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
@@ -2,29 +2,17 @@ using BCG;

 using bts;

-using Bts.DataStreamCore.StreamFactories;
-
-using Spark.Core.Disposables;
-
 using System;

 namespace BCG {
-   /// <summary>
-   /// This is a stream so we can run it from the DSQA (since dsqa can only load streams).
-   /// <para>
-   /// Note: It's hidden because this stream is now created by an IDataStreamFactory
-   /// (see RemoteQueryStreamFactory.cs for details)
-   /// </para>
-   /// </summary>
-   [DataStreamUserRespresentation(false)]
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
   public class RemoteQueryStream : ViewStream {
      private readonly string _name, _stream_exp;
-      readonly TableStreamFactory tableStreamFactory;
      private PublisherCore _publisher_core = new PublisherCore();
-      IDisposable? subscription;
+      private ViewStream _source_str;

-      public RemoteQueryStream(TableStreamFactory tableStreamFactory, params string[] lines) {
-         this.tableStreamFactory = tableStreamFactory;
+      public RemoteQueryStream(params string[] lines) {
         _stream_exp = get_expression(lines);
         _name = Guid.NewGuid().ToString();
         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
@@ -33,17 +21,17 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         Log.isnull(subscription);
         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
         byte[] rem_req_bytes = rem_req.serialize();
         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
-         var sourceStream = tableStreamFactory.Create(_name);
-         sourceStream.onChange += _handle_data;
-         subscription = Disposable.Create(() => sourceStream.onChange -= _handle_data);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
      }

-      protected override void myOnLastSubscriber() =>
-         Disposable.dispose(ref subscription);
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }

      static string get_expression(params string[] lines) {
         for (int i = 0; i < lines.Length; i++) {
@@ -1,51 +1,33 @@
 using System;
-using System.Collections.Generic;

-namespace BCG {   
+namespace BCG {
   public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_single_tok(string exp) {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
         string[] toks = exp.Split(':');
-         string col_name = toks[0].Trim();
-         string col_value = toks[1].Trim();
+         string col_name = toks[0];
+         string col_value = toks[1];
         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
         return Tuple.Create(col_name, tup.Item1, tup.Item2);
      }

-      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
-         string[] toks = exp.Split(",");
-         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
-         foreach (string s in toks) {
-            var tup = parse_single_tok(s.Trim());
-            rv.Add(tup);
-         }
-         return rv;
-      }
-
      private Tuple<string, Type, object> _new_col;
-      private List<Tuple<string, Type, object>> _new_col_map;
      private IBlueTableSchema _out_schema;

      public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col_map = parse_exp(col_def);
+         _new_col = parse_exp(col_def);
         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
         mySetRealSource(rs);
      }

      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null) {
-            _out_schema = input_schema;
-            foreach (var m in _new_col_map) {
-               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
-            }
-         }
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
         return _out_schema;
      }

      private IBlueRow _map(IBlueRow r) {
         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         foreach (var m in _new_col_map) {
-            rv.setValue(m.Item1, m.Item3);
-         }
+         rv.setValue(_new_col.Item1, _new_col.Item3);
         return rv;
      }
   }
@@ -1,42 +1,52 @@
 using System;

-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("oldCount")]
-   public class CountStream : ViewStream {
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
      private int myId = 0;
      private ViewStream myVS;

-      public CountStream(ViewStream vs) {
+      public CountStream (ViewStream vs)
+      {
         myVS = vs;
      }

-      protected override void myOnFirstSubscriber () =>
-         myVS.onChange += myHandleChanges;
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}

-      protected override void myOnLastSubscriber () =>
-         myVS.onChange -= myHandleChanges;
+      protected override void myOnFirstSubscriber () {
+         myVS.onChange += new ChangeDelegate(myHandleChanges);
+      }
+      protected override void myOnLastSubscriber () {
+         myVS.onChange -= new ChangeDelegate(myHandleChanges);
+      }

-      private void myHandleChanges(object newValue) {
-         myFireEvent(
+
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
            myLastFiredDataPacket =
               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
                  myLastFiredDataPacket.newRows)
            );
      }

-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
         return retVal;
      }

@@ -30,7 +30,7 @@ namespace BCG

        protected override void myOnFirstSubscriber()
        {
-            myToCrush.onChange += myToCrush_onChange;
+            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
        }

        protected override void myOnLastSubscriber()
@@ -15,11 +15,11 @@ namespace BCG
            
            //set up an anonymous delegate for each of the streams
            //we do this so that we can unsubscribe
-            myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
+            myDelegates = new ChangeDelegate[myToCrush.Length];
            for (int i = 0; i < myToCrush.Length; ++i)
            {
                int tempI = i;
-                myDelegates[tempI] = o => myHandleChange(tempI, ((DataPacket)o).newRows);
+                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
            }
        }

@@ -27,7 +27,7 @@ namespace BCG
        private ViewStream[] myToCrush;
        private ITableCrusher myCrusher;

-        private DataStreamChangeDelegate[] myDelegates;
+        private ChangeDelegate[] myDelegates;

        protected override void myOnFirstSubscriber()
        {
				`@@ -1 +0,0 @@`
				`PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins`