Adding logs to CME MD

To mount a custom entrypoint script, put it in the config profile dir
and define the following env vars:

    SPARKD__IMAGE=sparkd_symbols
    SPARKD__MAIN_ENTRYPOINT=/home/bts/.local/share/bts/sparkd/conf/<custom-entrypoint>

To invoke the "normal" sparkd entrypoint script from the mounted one:

    ./sparkd-entrypoint.sh

.dotnet-cli/.nuget/packages/microsoft.net.test.sdk/17.10.0/build/netcoreapp3.1/Microsoft.NET.Test.Sdk.Program.cs

@@ -1,4 +0,0 @@
-// <auto-generated> This file has been auto generated. </auto-generated>
-using System;
-[Microsoft.VisualStudio.TestPlatform.TestSDKAutoGeneratedCode]
-class AutoGeneratedProgram {static void Main(string[] args){}}

.env

@@ -29,6 +29,9 @@
 #     tools/deploy.sh --help
 #
 
+### Host/Runtime ###
+TZ=Etc/UTC
+
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
 #   src/server/sparkd/Dockerfile
@@ -38,6 +41,9 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
+# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
+# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
+# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
@@ -45,6 +51,8 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
+SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
+SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
 
 ### PubSubServer ###
 PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development

.gitignore

@@ -64,9 +64,3 @@ trace_data/
 *.lst
 .env.local
 .secrets
-.dotnet-cli/
-.dotnet-cli/
-.dotnet-cli /
-.dotnet/
-.codex_restore*.log
-src/cs/Bts.Spark.Core/.codexobj/

.vscode/launch.json

@@ -183,11 +183,10 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -203,71 +202,9 @@
         "program": "PubSubServer.exe",
       },
       "cwd": "build/Debug/bin/net8.0",
-      // "env": {
-      //   "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      // },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DSPublisher",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "DSPublisher",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
       },
-      "windows": {
-        "program": "DSPublisher.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DSPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DSPublisher",
-      },
-      "windows": {
-        "program": "DSPublisher.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PersistenceServer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "PersistenceServer",
-      },
-      "windows": {
-        "program": "PersistenceServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PersistenceServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PersistenceServer",
-      },
-      "windows": {
-        "program": "PersistenceServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -511,14 +448,15 @@
       "default": "Local",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
-      "description": "PubSub Configuration",
+      "description": "PubSubServer Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Local",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sdkDemoEnvironment",

compose.yaml

@@ -2,6 +2,7 @@ services:
   sparkd:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -9,24 +10,35 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-shell:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+    build:
+      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: "do sleep 1; done"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -42,6 +54,7 @@ services:
   sparkd-symbols:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
       - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
@@ -50,26 +63,8 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/.local/share/bts/sparkd/conf/
-        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -77,6 +72,33 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+  sparkd-multicast:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-2048mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       restart_policy:
         condition: any
@@ -84,7 +106,6 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet8
     volumes:
@@ -195,3 +216,6 @@ volumes:
     name: "md_publisher_22019"
   historic_md:
     name: "historic_md"
+  md_recording:
+    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
+    external: true

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y ca-certificates
 RUN apt-get update && apt-get install -y libssh2-1-dev
 RUN apt-get update && apt-get install -y software-properties-common
 RUN apt-get update && apt-get install -y sudo
+RUN apt-get update && apt-get install -y tzdata
 
 # GCC
 ARG gcc_major=13

docker/images/bluetradingsystems/linux_shell/Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:24.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+  fish \
+  less \
+  vim
+
+USER bts
+WORKDIR /home/bts

spark_client.sln

@@ -1,433 +1,306 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
+EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{1A5B8C5D-BADC-457E-819F-0050C44FE4D3}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{2A0FBDE0-D870-4567-85BD-F7793A5862C6}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
 		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
 		Debug|x86 = Debug|x86
-		MinSizeRel|x64 = MinSizeRel|x64
-		MinSizeRel|Any CPU = MinSizeRel|Any CPU
-		MinSizeRel|x86 = MinSizeRel|x86
-		Release|x64 = Release|x64
 		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
 		Release|x86 = Release|x86
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
-		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.Build.0 = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.ActiveCfg = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.Build.0 = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.Build.0 = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.Build.0 = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.ActiveCfg = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.Build.0 = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.Build.0 = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.ActiveCfg = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.Build.0 = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.Build.0 = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.ActiveCfg = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.Build.0 = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
+		{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{1A5B8C5D-BADC-457E-819F-0050C44FE4D3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6} = {1A5B8C5D-BADC-457E-819F-0050C44FE4D3}
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-   "dotnet.preferCSharpExtension": true
-}

src/DataStreamSuite/BlueOrbDSL/DSQA_LAST_EXECUTED_QUERY.txt

@@ -0,0 +1 @@
+r=TimeSeriesStream(ReplayFutureLiveBBOStream("ES"))

src/DataStreamSuite/BlueOrbDSL/DSQA_RESULTS_EXPORT_20260216_195816.csv

@@ -0,0 +1,3 @@
+x,y,z,id
+123,1.001,abc,1
+123,1.001,abc,0

src/DataStreamSuite/BlueOrbDSL/DSQA_RESULTS_EXPORT_20260216_195818.csv

@@ -0,0 +1,3 @@
+x,y,z,id
+123,1.001,abc,1
+123,1.001,abc,0

src/DataStreamSuite/BlueOrbDSL/DSQA_RESULTS_EXPORT_20260313_055725.csv

@@ -0,0 +1,78 @@
+id,ts_event,contract,bid_price,bid_size,ask_price,ask_size,timestamp
+76,2026-03-13T05:57:25.177965500Z,ESU5,6363.75,51,6364,63,3/13/2026 5:57:25 AM
+75,2026-03-13T05:57:25.068062900Z,ESU5,6363.75,52,6364,63,3/13/2026 5:57:25 AM
+74,2026-03-13T05:57:24.963428500Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+73,2026-03-13T05:57:24.781215700Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+72,2026-03-13T05:57:24.676290100Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+71,2026-03-13T05:57:24.566898400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+70,2026-03-13T05:57:24.430800400Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+69,2026-03-13T05:57:24.273683400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+68,2026-03-13T05:57:24.167656100Z,ESU5,6363.75,56,6364,64,3/13/2026 5:57:24 AM
+67,2026-03-13T05:57:23.995720100Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:23 AM
+66,2026-03-13T05:57:23.894029100Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
+65,2026-03-13T05:57:23.728897300Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
+64,2026-03-13T05:57:23.614316800Z,ESU5,6363.75,52,6364,68,3/13/2026 5:57:23 AM
+63,2026-03-13T05:57:23.503752700Z,ESU5,6363.75,51,6364,68,3/13/2026 5:57:23 AM
+62,2026-03-13T05:57:23.289108200Z,ESU5,6363.75,50,6364,68,3/13/2026 5:57:23 AM
+61,2026-03-13T05:57:23.155673400Z,ESU5,6363.75,49,6364,68,3/13/2026 5:57:23 AM
+60,2026-03-13T05:57:23.028898400Z,ESU5,6363.75,48,6364,68,3/13/2026 5:57:23 AM
+59,2026-03-13T05:57:22.887132700Z,ESU5,6363.75,46,6364,68,3/13/2026 5:57:22 AM
+58,2026-03-13T05:57:22.676916300Z,ESU5,6363.75,44,6364,68,3/13/2026 5:57:22 AM
+57,2026-03-13T05:57:22.520328000Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:22 AM
+56,2026-03-13T05:57:22.405244600Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:22 AM
+55,2026-03-13T05:57:22.294674100Z,ESU5,6363.75,41,6364,69,3/13/2026 5:57:22 AM
+54,2026-03-13T05:57:22.163467900Z,ESU5,6363.75,38,6364,69,3/13/2026 5:57:22 AM
+53,2026-03-13T05:57:21.992443000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
+52,2026-03-13T05:57:21.872338400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
+51,2026-03-13T05:57:21.742801800Z,ESU5,6363.75,37,6364,71,3/13/2026 5:57:21 AM
+50,2026-03-13T05:57:21.557711100Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
+49,2026-03-13T05:57:21.434977000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
+48,2026-03-13T05:57:21.285751600Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
+47,2026-03-13T05:57:21.184392000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
+46,2026-03-13T05:57:21.076843800Z,ESU5,6363.75,34,6364,73,3/13/2026 5:57:21 AM
+45,2026-03-13T05:57:20.894131800Z,ESU5,6363.75,32,6364,73,3/13/2026 5:57:20 AM
+44,2026-03-13T05:57:20.759550000Z,ESU5,6363.75,30,6364,73,3/13/2026 5:57:20 AM
+43,2026-03-13T05:57:20.641547900Z,ESU5,6363.75,28,6364,73,3/13/2026 5:57:20 AM
+42,2026-03-13T05:57:20.540293200Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
+41,2026-03-13T05:57:20.439034600Z,ESU5,6363.75,27,6364,74,3/13/2026 5:57:20 AM
+40,2026-03-13T05:57:20.337795000Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
+39,2026-03-13T05:57:20.236606600Z,ESU5,6363.75,27,6364,72,3/13/2026 5:57:20 AM
+38,2026-03-13T05:57:20.135417700Z,ESU5,6363.75,30,6364,72,3/13/2026 5:57:20 AM
+37,2026-03-13T05:57:20.034272700Z,ESU5,6363.75,32,6364,72,3/13/2026 5:57:20 AM
+36,2026-03-13T05:57:19.933096800Z,ESU5,6363.75,34,6364,72,3/13/2026 5:57:19 AM
+35,2026-03-13T05:57:19.832012600Z,ESU5,6363.75,36,6364,72,3/13/2026 5:57:19 AM
+34,2026-03-13T05:57:19.730878500Z,ESU5,6363.75,38,6364,72,3/13/2026 5:57:19 AM
+33,2026-03-13T05:57:19.629787400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
+32,2026-03-13T05:57:19.528697000Z,ESU5,6363.75,38,6364,70,3/13/2026 5:57:19 AM
+31,2026-03-13T05:57:19.427077000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
+30,2026-03-13T05:57:19.326450300Z,ESU5,6363.75,39,6364,71,3/13/2026 5:57:19 AM
+29,2026-03-13T05:57:19.225427900Z,ESU5,6363.75,39,6364,70,3/13/2026 5:57:19 AM
+28,2026-03-13T05:57:19.123339800Z,ESU5,6363.75,40,6364,70,3/13/2026 5:57:19 AM
+27,2026-03-13T05:57:19.021936500Z,ESU5,6363.75,41,6364,70,3/13/2026 5:57:19 AM
+26,2026-03-13T05:57:18.920859200Z,ESU5,6363.75,45,6364,70,3/13/2026 5:57:18 AM
+25,2026-03-13T05:57:18.820148400Z,ESU5,6363.75,46,6364,70,3/13/2026 5:57:18 AM
+24,2026-03-13T05:57:18.718988100Z,ESU5,6363.75,46,6364,69,3/13/2026 5:57:18 AM
+23,2026-03-13T05:57:18.618299700Z,ESU5,6363.75,49,6364,69,3/13/2026 5:57:18 AM
+22,2026-03-13T05:57:18.517138100Z,ESU5,6363.75,52,6364,69,3/13/2026 5:57:18 AM
+21,2026-03-13T05:57:18.416335200Z,ESU5,6363.75,53,6364,69,3/13/2026 5:57:18 AM
+20,2026-03-13T05:57:18.315335900Z,ESU5,6363.75,55,6364,69,3/13/2026 5:57:18 AM
+19,2026-03-13T05:57:18.214368600Z,ESU5,6363.75,57,6364,69,3/13/2026 5:57:18 AM
+18,2026-03-13T05:57:18.113754700Z,ESU5,6363.75,57,6364,65,3/13/2026 5:57:18 AM
+17,2026-03-13T05:57:18.012706500Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:18 AM
+16,2026-03-13T05:57:17.911672600Z,ESU5,6363.75,54,6364,65,3/13/2026 5:57:17 AM
+15,2026-03-13T05:57:17.810879900Z,ESU5,6363.75,52,6364,65,3/13/2026 5:57:17 AM
+14,2026-03-13T05:57:17.710089700Z,ESU5,6363.75,49,6364,65,3/13/2026 5:57:17 AM
+13,2026-03-13T05:57:17.609388300Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+12,2026-03-13T05:57:17.508697400Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+11,2026-03-13T05:57:17.408027000Z,ESU5,6363.75,53,6364,67,3/13/2026 5:57:17 AM
+10,2026-03-13T05:57:17.307406200Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+9,2026-03-13T05:57:17.206714300Z,ESU5,6363.75,49,6364,71,3/13/2026 5:57:17 AM
+8,2026-03-13T05:57:17.105530300Z,ESU5,6363.75,48,6364,71,3/13/2026 5:57:17 AM
+7,2026-03-13T05:57:17.004796800Z,ESU5,6363.75,47,6364,71,3/13/2026 5:57:17 AM
+6,2026-03-13T05:57:16.904113900Z,ESU5,6363.75,46,6364,71,3/13/2026 5:57:16 AM
+5,2026-03-13T05:57:16.803381900Z,ESU5,6363.75,43,6364,71,3/13/2026 5:57:16 AM
+4,2026-03-13T05:57:16.701969600Z,ESU5,6363.75,43,6364,72,3/13/2026 5:57:16 AM
+3,2026-03-13T05:57:16.600765600Z,ESU5,6363.75,43,6364,73,3/13/2026 5:57:16 AM
+2,2026-03-13T05:57:16.499986300Z,ESU5,6363.75,43,6364,74,3/13/2026 5:57:16 AM
+1,2026-03-13T05:57:16.399241700Z,ESU5,6363.75,43,6364,75,3/13/2026 5:57:16 AM
+0,2026-03-13T05:57:16.264038000Z,ESU5,6363.75,43,6364,76,3/13/2026 5:57:16 AM

src/DataStreamSuite/BlueOrbDSL/DSQA_RESULTS_EXPORT_20260313_055727.csv

@@ -0,0 +1,91 @@
+id,ts_event,contract,bid_price,bid_size,ask_price,ask_size,timestamp
+89,2026-03-13T05:57:26.904401600Z,ESU5,6363.75,43,6364,67,3/13/2026 5:57:26 AM
+88,2026-03-13T05:57:26.778037200Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:26 AM
+87,2026-03-13T05:57:26.597038500Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:26 AM
+86,2026-03-13T05:57:26.438768300Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:26 AM
+85,2026-03-13T05:57:26.328259100Z,ESU5,6363.75,43,6364,67,3/13/2026 5:57:26 AM
+84,2026-03-13T05:57:26.216211200Z,ESU5,6363.75,44,6364,67,3/13/2026 5:57:26 AM
+83,2026-03-13T05:57:26.099281200Z,ESU5,6363.75,46,6364,67,3/13/2026 5:57:26 AM
+82,2026-03-13T05:57:25.979814900Z,ESU5,6363.75,48,6364,67,3/13/2026 5:57:25 AM
+81,2026-03-13T05:57:25.852317700Z,ESU5,6363.75,48,6364,66,3/13/2026 5:57:25 AM
+80,2026-03-13T05:57:25.684323100Z,ESU5,6363.75,48,6364,65,3/13/2026 5:57:25 AM
+79,2026-03-13T05:57:25.539436900Z,ESU5,6363.75,48,6364,64,3/13/2026 5:57:25 AM
+78,2026-03-13T05:57:25.425439200Z,ESU5,6363.75,49,6364,64,3/13/2026 5:57:25 AM
+77,2026-03-13T05:57:25.298054100Z,ESU5,6363.75,51,6364,64,3/13/2026 5:57:25 AM
+76,2026-03-13T05:57:25.177965500Z,ESU5,6363.75,51,6364,63,3/13/2026 5:57:25 AM
+75,2026-03-13T05:57:25.068062900Z,ESU5,6363.75,52,6364,63,3/13/2026 5:57:25 AM
+74,2026-03-13T05:57:24.963428500Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+73,2026-03-13T05:57:24.781215700Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+72,2026-03-13T05:57:24.676290100Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+71,2026-03-13T05:57:24.566898400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+70,2026-03-13T05:57:24.430800400Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+69,2026-03-13T05:57:24.273683400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+68,2026-03-13T05:57:24.167656100Z,ESU5,6363.75,56,6364,64,3/13/2026 5:57:24 AM
+67,2026-03-13T05:57:23.995720100Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:23 AM
+66,2026-03-13T05:57:23.894029100Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
+65,2026-03-13T05:57:23.728897300Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
+64,2026-03-13T05:57:23.614316800Z,ESU5,6363.75,52,6364,68,3/13/2026 5:57:23 AM
+63,2026-03-13T05:57:23.503752700Z,ESU5,6363.75,51,6364,68,3/13/2026 5:57:23 AM
+62,2026-03-13T05:57:23.289108200Z,ESU5,6363.75,50,6364,68,3/13/2026 5:57:23 AM
+61,2026-03-13T05:57:23.155673400Z,ESU5,6363.75,49,6364,68,3/13/2026 5:57:23 AM
+60,2026-03-13T05:57:23.028898400Z,ESU5,6363.75,48,6364,68,3/13/2026 5:57:23 AM
+59,2026-03-13T05:57:22.887132700Z,ESU5,6363.75,46,6364,68,3/13/2026 5:57:22 AM
+58,2026-03-13T05:57:22.676916300Z,ESU5,6363.75,44,6364,68,3/13/2026 5:57:22 AM
+57,2026-03-13T05:57:22.520328000Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:22 AM
+56,2026-03-13T05:57:22.405244600Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:22 AM
+55,2026-03-13T05:57:22.294674100Z,ESU5,6363.75,41,6364,69,3/13/2026 5:57:22 AM
+54,2026-03-13T05:57:22.163467900Z,ESU5,6363.75,38,6364,69,3/13/2026 5:57:22 AM
+53,2026-03-13T05:57:21.992443000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
+52,2026-03-13T05:57:21.872338400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
+51,2026-03-13T05:57:21.742801800Z,ESU5,6363.75,37,6364,71,3/13/2026 5:57:21 AM
+50,2026-03-13T05:57:21.557711100Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
+49,2026-03-13T05:57:21.434977000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
+48,2026-03-13T05:57:21.285751600Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
+47,2026-03-13T05:57:21.184392000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
+46,2026-03-13T05:57:21.076843800Z,ESU5,6363.75,34,6364,73,3/13/2026 5:57:21 AM
+45,2026-03-13T05:57:20.894131800Z,ESU5,6363.75,32,6364,73,3/13/2026 5:57:20 AM
+44,2026-03-13T05:57:20.759550000Z,ESU5,6363.75,30,6364,73,3/13/2026 5:57:20 AM
+43,2026-03-13T05:57:20.641547900Z,ESU5,6363.75,28,6364,73,3/13/2026 5:57:20 AM
+42,2026-03-13T05:57:20.540293200Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
+41,2026-03-13T05:57:20.439034600Z,ESU5,6363.75,27,6364,74,3/13/2026 5:57:20 AM
+40,2026-03-13T05:57:20.337795000Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
+39,2026-03-13T05:57:20.236606600Z,ESU5,6363.75,27,6364,72,3/13/2026 5:57:20 AM
+38,2026-03-13T05:57:20.135417700Z,ESU5,6363.75,30,6364,72,3/13/2026 5:57:20 AM
+37,2026-03-13T05:57:20.034272700Z,ESU5,6363.75,32,6364,72,3/13/2026 5:57:20 AM
+36,2026-03-13T05:57:19.933096800Z,ESU5,6363.75,34,6364,72,3/13/2026 5:57:19 AM
+35,2026-03-13T05:57:19.832012600Z,ESU5,6363.75,36,6364,72,3/13/2026 5:57:19 AM
+34,2026-03-13T05:57:19.730878500Z,ESU5,6363.75,38,6364,72,3/13/2026 5:57:19 AM
+33,2026-03-13T05:57:19.629787400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
+32,2026-03-13T05:57:19.528697000Z,ESU5,6363.75,38,6364,70,3/13/2026 5:57:19 AM
+31,2026-03-13T05:57:19.427077000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
+30,2026-03-13T05:57:19.326450300Z,ESU5,6363.75,39,6364,71,3/13/2026 5:57:19 AM
+29,2026-03-13T05:57:19.225427900Z,ESU5,6363.75,39,6364,70,3/13/2026 5:57:19 AM
+28,2026-03-13T05:57:19.123339800Z,ESU5,6363.75,40,6364,70,3/13/2026 5:57:19 AM
+27,2026-03-13T05:57:19.021936500Z,ESU5,6363.75,41,6364,70,3/13/2026 5:57:19 AM
+26,2026-03-13T05:57:18.920859200Z,ESU5,6363.75,45,6364,70,3/13/2026 5:57:18 AM
+25,2026-03-13T05:57:18.820148400Z,ESU5,6363.75,46,6364,70,3/13/2026 5:57:18 AM
+24,2026-03-13T05:57:18.718988100Z,ESU5,6363.75,46,6364,69,3/13/2026 5:57:18 AM
+23,2026-03-13T05:57:18.618299700Z,ESU5,6363.75,49,6364,69,3/13/2026 5:57:18 AM
+22,2026-03-13T05:57:18.517138100Z,ESU5,6363.75,52,6364,69,3/13/2026 5:57:18 AM
+21,2026-03-13T05:57:18.416335200Z,ESU5,6363.75,53,6364,69,3/13/2026 5:57:18 AM
+20,2026-03-13T05:57:18.315335900Z,ESU5,6363.75,55,6364,69,3/13/2026 5:57:18 AM
+19,2026-03-13T05:57:18.214368600Z,ESU5,6363.75,57,6364,69,3/13/2026 5:57:18 AM
+18,2026-03-13T05:57:18.113754700Z,ESU5,6363.75,57,6364,65,3/13/2026 5:57:18 AM
+17,2026-03-13T05:57:18.012706500Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:18 AM
+16,2026-03-13T05:57:17.911672600Z,ESU5,6363.75,54,6364,65,3/13/2026 5:57:17 AM
+15,2026-03-13T05:57:17.810879900Z,ESU5,6363.75,52,6364,65,3/13/2026 5:57:17 AM
+14,2026-03-13T05:57:17.710089700Z,ESU5,6363.75,49,6364,65,3/13/2026 5:57:17 AM
+13,2026-03-13T05:57:17.609388300Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+12,2026-03-13T05:57:17.508697400Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+11,2026-03-13T05:57:17.408027000Z,ESU5,6363.75,53,6364,67,3/13/2026 5:57:17 AM
+10,2026-03-13T05:57:17.307406200Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+9,2026-03-13T05:57:17.206714300Z,ESU5,6363.75,49,6364,71,3/13/2026 5:57:17 AM
+8,2026-03-13T05:57:17.105530300Z,ESU5,6363.75,48,6364,71,3/13/2026 5:57:17 AM
+7,2026-03-13T05:57:17.004796800Z,ESU5,6363.75,47,6364,71,3/13/2026 5:57:17 AM
+6,2026-03-13T05:57:16.904113900Z,ESU5,6363.75,46,6364,71,3/13/2026 5:57:16 AM
+5,2026-03-13T05:57:16.803381900Z,ESU5,6363.75,43,6364,71,3/13/2026 5:57:16 AM
+4,2026-03-13T05:57:16.701969600Z,ESU5,6363.75,43,6364,72,3/13/2026 5:57:16 AM
+3,2026-03-13T05:57:16.600765600Z,ESU5,6363.75,43,6364,73,3/13/2026 5:57:16 AM
+2,2026-03-13T05:57:16.499986300Z,ESU5,6363.75,43,6364,74,3/13/2026 5:57:16 AM
+1,2026-03-13T05:57:16.399241700Z,ESU5,6363.75,43,6364,75,3/13/2026 5:57:16 AM
+0,2026-03-13T05:57:16.264038000Z,ESU5,6363.75,43,6364,76,3/13/2026 5:57:16 AM

src/DataStreamSuite/BlueOrbDSL/__init__.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+import os
+import subprocess
+import threading
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class DataPacket:
+    state: List[Any]
+    deleted_ids: List[int]
+    added_ids: List[int]
+
+
+@dataclass
+class DslResponse:
+    data_packet: DataPacket | None
+    req_id: str | None
+    error: str | None
+
+
+def _resolve_interop_exe() -> str:
+    env_path = os.environ.get("BLUEORB_DSL_INTEROP")
+    if env_path and os.path.isfile(env_path):
+        return env_path
+    candidates = [
+        "/workspaces/spark/build/Debug/bin/net8.0/BlueOrbDSLInteropServer",
+        "/workspaces/spark/build/Release/bin/net8.0/BlueOrbDSLInteropServer",
+    ]
+    for path in candidates:
+        if os.path.isfile(path):
+            return path
+    raise FileNotFoundError(
+        "BlueOrbDSLInterop executable not found. "
+        "Set BLUEORB_DSL_INTEROP to the full path."
+    )
+
+_interop_proc: Optional[subprocess.Popen] = None
+_reader_thread: Optional[threading.Thread] = None
+_callbacks: Dict[str, Callable[[DslResponse], None]] = {}
+_first_response_events: Dict[str, threading.Event] = {}
+_first_responses: Dict[str, DslResponse] = {}
+_state_lock = threading.Lock()
+
+
+def start_interop() -> None:
+    """
+    Start the BlueOrbDSLInteropServer process if it's not already running.
+    """
+    global _interop_proc
+    if _interop_proc is not None and _interop_proc.poll() is None:
+        return
+    exe = _resolve_interop_exe()
+    _interop_proc = subprocess.Popen(
+        [exe],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    _start_reader_thread()
+
+
+def stop_interop() -> None:
+    """
+    Stop the BlueOrbDSLInteropServer process if it's running.
+    """
+    global _interop_proc
+    if _interop_proc is None:
+        return
+    if _interop_proc.poll() is None and _interop_proc.stdin is not None:
+        _interop_proc.stdin.write(":exit\n")
+        _interop_proc.stdin.flush()
+        _interop_proc.stdin.close()
+        _interop_proc.wait(timeout=5)
+    global _reader_thread
+    if _reader_thread is not None:
+        _reader_thread.join(timeout=1)
+        _reader_thread = None
+    with _state_lock:
+        _callbacks.clear()
+        _first_response_events.clear()
+        _first_responses.clear()
+    _interop_proc = None
+
+
+def get_interop_status() -> tuple[bool, int | None]:
+    """
+    Returns current interop process status as (is_running, exit_code).
+    exit_code is None when process was never started or is still running.
+    """
+    proc = _interop_proc
+    if proc is None:
+        return (False, None)
+    rc = proc.poll()
+    return (rc is None, rc)
+
+
+def get_dsl(dsl: str, req_id: str, callback: Callable[[DslResponse], None]) -> DslResponse:
+    """
+    Sends the DSL to BlueOrbDSLInteropServer and returns the first response for this req_id.
+    Subsequent streaming updates for the same req_id are delivered via callback.
+    Requires start_interop() to be called first.
+    """
+    assert _interop_proc is not None and _interop_proc.poll() is None, (
+        "Interop process is not running. Call start_interop() first."
+    )
+    assert _interop_proc.stdin is not None
+
+    payload = {"dsl": dsl, "req_id": req_id}
+    first_event = threading.Event()
+    with _state_lock:
+        _callbacks[req_id] = callback
+        _first_response_events[req_id] = first_event
+        _first_responses.pop(req_id, None)
+
+    _interop_proc.stdin.write(json.dumps(payload) + "\n")
+    _interop_proc.stdin.flush()
+
+    if not first_event.wait(timeout=10):
+        with _state_lock:
+            _first_response_events.pop(req_id, None)
+            _callbacks.pop(req_id, None)
+        raise TimeoutError(f"No response received for req_id '{req_id}' from BlueOrbDSLInterop.")
+
+    with _state_lock:
+        response = _first_responses.pop(req_id, None)
+        _first_response_events.pop(req_id, None)
+    if response is None:
+        raise RuntimeError(f"Missing first response for req_id '{req_id}'.")
+    return response
+
+
+def _start_reader_thread() -> None:
+    global _reader_thread
+    if _reader_thread is not None and _reader_thread.is_alive():
+        return
+    _reader_thread = threading.Thread(target=_reader_loop, daemon=True)
+    _reader_thread.start()
+
+
+def _reader_loop() -> None:
+    while True:
+        proc = _interop_proc
+        if proc is None or proc.poll() is not None:
+            return
+        assert proc.stdout is not None
+        raw = proc.stdout.readline()
+        if raw == "":
+            return
+
+        line = raw.strip()
+        if not line:
+            continue
+
+        try:
+            payload = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        response = DslResponse(
+            data_packet=DataPacket(state=[payload.get("data_packet")], deleted_ids=[], added_ids=[]),
+            req_id=payload.get("req_id"),
+            error=payload.get("error"),
+        )
+        req_id = response.req_id
+        if not req_id:
+            continue
+
+        callback: Optional[Callable[[DslResponse], None]]
+        event: Optional[threading.Event]
+        with _state_lock:
+            callback = _callbacks.get(req_id)
+            event = _first_response_events.get(req_id)
+            if event is not None and req_id not in _first_responses:
+                _first_responses[req_id] = response
+                event.set()
+
+        if callback is not None:
+            try:
+                callback(response)
+            except Exception:
+                # Keep the reader alive if callback logic fails.
+                pass

src/DataStreamSuite/BlueOrbDSL/run_blueorb_dsl.py

@@ -0,0 +1,33 @@
+import os
+import sys
+import uuid
+
+_here = os.path.dirname(os.path.abspath(__file__))
+_parent = os.path.dirname(_here)
+if _parent not in sys.path:
+    sys.path.insert(0, _parent)
+
+from BlueOrbDSL import get_dsl, start_interop, stop_interop, DslResponse
+
+
+def on_packet(response: DslResponse) -> None:
+    print("callback from DSL Server, DslResponse:", response)
+
+
+def main() -> None:
+    start_interop()
+    try:
+        while True:
+            dsl = input("Enter DSL (or 'quit'): ").strip()
+            if dsl.lower() == "quit":
+                break
+            if not dsl:
+                continue
+            response = get_dsl(dsl, str(uuid.uuid4()), on_packet)
+            print("return:", response)
+    finally:
+        stop_interop()
+
+
+if __name__ == "__main__":
+    main()

src/DataStreamSuite/BlueOrbDSL/run_dsqa.py

@@ -0,0 +1,844 @@
+from __future__ import annotations
+
+import csv
+import time
+import os
+import sys
+import threading
+import uuid
+from collections import deque
+from datetime import datetime
+from typing import Any
+
+from rich.text import Text
+from rich.style import Style
+from textual import events
+from textual.app import App, ComposeResult
+from textual.binding import Binding
+from textual.containers import Container, Horizontal, Vertical
+from textual.widgets.text_area import TextAreaTheme
+from textual.widgets import Button, DataTable, Input, Static, TabbedContent, TabPane, TextArea, Tree
+
+_here = os.path.dirname(os.path.abspath(__file__))
+_parent = os.path.dirname(_here)
+if _parent not in sys.path:
+    sys.path.insert(0, _parent)
+
+from BlueOrbDSL import DslResponse, get_dsl, get_interop_status, start_interop, stop_interop
+
+
+class DSQALiveConsole(App):
+    DEFAULT_QUERY = (
+        'FROM market.ticks\n'
+        'WHERE symbol IN ("BTCUSD", "ETHUSD")\n'
+        "SELECT symbol, price, change, time"
+    )
+    LAST_QUERY_PATH = os.path.join(_here, "DSQA_LAST_EXECUTED_QUERY.txt")
+    DSL_HIGHLIGHT_QUERY = r"""
+    ; comments
+    (comment) @comment
+
+    ; assignment vars: lhs in x = SomeStream(...)
+    (assignment_expression
+      left: (identifier) @variable)
+
+    ; stream operators (function names ending with Stream)
+    (call_expression
+      function: (identifier) @function
+      (#match? @function "^[A-Za-z_][A-Za-z0-9_]*Stream$"))
+
+    ; normal function calls
+    (call_expression
+      function: (identifier) @function.method)
+
+    ; arguments and literals
+    (arguments (identifier) @parameter)
+    (string) @string
+    (number) @number
+    (true) @constant.builtin
+    (false) @constant.builtin
+    (null) @constant.builtin
+    """
+
+    BINDINGS = [
+        Binding("ctrl+r", "execute", "Execute"),
+        Binding("f11", "execute", show=False),
+        Binding("alt+enter", "execute", show=False),
+        # Override Textual's default Ctrl+Q quit to avoid accidental VS Code close.
+        Binding("ctrl+q", "noop", show=False),
+        Binding("ctrl+x", "quit", "Quit"),
+    ]
+    CSS = """
+    Screen {
+        background: #060a1c;
+        color: #d8dff4;
+    }
+
+    #window_close_button {
+        width: 3;
+        min-width: 3;
+        height: 1;
+        margin: 0;
+        border: none;
+        background: #3a1015;
+        color: #ff9ca8;
+        text-style: bold;
+    }
+
+    #window_close_button:hover,
+    #window_close_button:focus {
+        background: #5a1620;
+        color: #ffd6db;
+    }
+
+    #root {
+        padding: 0 1 0 1;
+        height: 1fr;
+    }
+
+    #query_panel {
+        background: #070d27;
+        height: 33%;
+        margin-top: 1;
+    }
+
+    #query_header_row {
+        height: 1;
+        border-bottom: solid #596285;
+    }
+
+    #query_header {
+        width: auto;
+        color: #9eb0d4;
+        content-align: left middle;
+    }
+
+    #query_header_spacer {
+        width: 1fr;
+    }
+
+    #query_input {
+        border: solid #697597;
+        background: #050b22;
+        color: #cfd7eb;
+        height: 1fr;
+        padding-left: 1;
+    }
+
+    #results_panel {
+        background: #070d27;
+        height: 1fr;
+        margin-top: 1;
+    }
+
+    #results_toolbar {
+        height: 3;
+        padding: 0 1;
+    }
+
+    #results_left {
+        width: auto;
+        height: 3;
+        margin-top: 0;
+    }
+
+    #stream_toggle_button {
+        width: auto;
+        min-width: 11;
+        margin-top: 0;
+        margin-left: 1;
+        border: none;
+        background: #070d27;
+        color: #79afe7;
+    }
+
+    #stream_toggle_button:hover,
+    #stream_toggle_button:focus {
+        background: #111a3f;
+        color: #9bc9f6;
+    }
+
+    #execute_button {
+        width: 5;
+        min-width: 5;
+        margin-top: 0;
+        margin-left: 1;
+        border: none;
+        background: #10341f;
+        color: #7ef1ac;
+        text-style: bold;
+    }
+
+    #execute_button:hover,
+    #execute_button:focus {
+        background: #1a4f30;
+        color: #b3ffd0;
+    }
+
+    #results_actions {
+        width: auto;
+        height: 3;
+        margin-top: 0;
+    }
+
+    #results_spacer {
+        width: 1fr;
+    }
+
+    #results_search {
+        width: 30;
+        min-width: 24;
+        margin-right: 1;
+        border: none;
+        background: #060d28;
+        color: #d5def3;
+        padding: 0 1;
+    }
+
+    #search_button,
+    #copy_button,
+    #export_button {
+        width: auto;
+        min-width: 11;
+        margin-left: 1;
+        border: none;
+        background: #070d27;
+        color: #79afe7;
+    }
+
+    #search_button:hover,
+    #copy_button:hover,
+    #export_button:hover,
+    #search_button:focus,
+    #copy_button:focus,
+    #export_button:focus {
+        background: #111a3f;
+        color: #9bc9f6;
+    }
+
+    #results_table {
+        border: solid #697597;
+        background: #050b22;
+        height: 1fr;
+    }
+
+    #results_table .datatable--header,
+    #results_table .datatable--header-hover,
+    #results_table .datatable--header-cursor {
+        background: #0c1535;
+        color: #6ea6df;
+        text-style: bold;
+    }
+
+    #results_table .datatable--even-row {
+        background: #050b22;
+    }
+
+    #results_table .datatable--odd-row {
+        background: #07102a;
+    }
+
+    #results_table .datatable--cursor {
+        background: #19335b;
+        color: #dbe8ff;
+    }
+
+    #results_table .datatable--fixed-cursor {
+        background: #132949;
+        color: #c8dbff;
+    }
+
+    #results_table.error .datatable--odd-row,
+    #results_table.error .datatable--even-row,
+    #results_table.error .datatable--cursor,
+    #results_table.error .datatable--fixed-cursor {
+        color: #f17486;
+        text-style: bold;
+    }
+
+    #results_panel.paused {
+        background: #150f1f;
+    }
+
+    #results_table.paused {
+        background: #130f18;
+    }
+
+    #results_table.paused .datatable--header,
+    #results_table.paused .datatable--header-hover,
+    #results_table.paused .datatable--header-cursor {
+        background: #262338;
+        color: #8b90a1;
+        text-style: bold;
+    }
+
+    #results_tabs {
+        height: 1fr;
+        background: #070d27;
+    }
+
+    #data_tab,
+    #tree_tab {
+        padding: 0;
+    }
+
+    #ops_tree {
+        border: solid #697597;
+        background: #050b22;
+        color: #cfd7eb;
+        height: 1fr;
+    }
+
+    #bottom_status_bar {
+        border-top: solid #596285;
+        background: #070d27;
+        height: 2;
+        margin: 0;
+        padding: 0 1;
+    }
+
+    #status_right_wrap {
+        width: 1fr;
+    }
+
+    #status_row {
+        height: 1;
+        margin-top: 0;
+        width: 1fr;
+    }
+
+    #live_status {
+        width: 1fr;
+        color: #b3bed9;
+        content-align: right middle;
+    }
+    """
+
+    def compose(self) -> ComposeResult:
+        with Container(id="root"):
+            with Vertical(id="query_panel"):
+                with Horizontal(id="query_header_row"):
+                    yield Static("DSL Query", id="query_header")
+                    yield Static("", id="query_header_spacer")
+                    yield Button("✕", id="window_close_button")
+                yield TextArea(
+                    text=self._load_last_query(),
+                    id="query_input",
+                    language="javascript",
+                    theme="vscode_dark",
+                    show_line_numbers=True,
+                )
+            with Vertical(id="results_panel"):
+                with Horizontal(id="results_toolbar"):
+                    with Horizontal(id="results_left"):
+                        yield Button("▶", id="execute_button")
+                        yield Button("⏸ Pause", id="stream_toggle_button")
+                    yield Static("", id="results_spacer")
+                    with Horizontal(id="results_actions"):
+                        yield Input(placeholder="Search: col:value", id="results_search")
+                        yield Button("⌕ Search", id="search_button")
+                        yield Button("⧉ Copy", id="copy_button")
+                        yield Button("⇩ Export", id="export_button")
+                with TabbedContent(initial="data_tab", id="results_tabs"):
+                    with TabPane("Data", id="data_tab"):
+                        table = DataTable(id="results_table")
+                        table.cursor_type = "cell"
+                        table.show_cursor = True
+                        table.zebra_stripes = True
+                        yield table
+                    with TabPane("Operators", id="tree_tab"):
+                        tree = Tree("Assemblies", id="ops_tree")
+                        tree.show_root = False
+                        yield tree
+        with Horizontal(id="bottom_status_bar"):
+            with Horizontal(id="status_right_wrap"):
+                with Horizontal(id="status_row"):
+                    yield Static(
+                        "Rows: 0 | Rate: 0.0/s | Total Updates: 0 | Uptime: 00:00:00 | Last Update: --:--:--",
+                        id="live_status",
+                    )
+
+    def on_mount(self) -> None:
+        self._update_timestamps: deque[float] = deque()
+        self._last_update_at: datetime | None = None
+        self._stream_started_at: float | None = None
+        self._total_updates = 0
+        self._last_row_count = 0
+        self._current_col_names: list[str] = []
+        self._current_rows: list[dict[str, Any]] = []
+        self._column_width_cache: dict[str, int] = {}
+        self._ops_catalog_rows: list[dict[str, Any]] = []
+        self._ops_catalog_error: str | None = None
+        self._ops_catalog_loaded = False
+        self._ops_catalog_loading = False
+        self._interop_is_running = False
+        self._interop_exit_code: int | None = None
+        self._grid_updates_enabled = True
+        self._paused_latest_response: DslResponse | None = None
+        self._ensure_interop_started()
+        self._refresh_ops_tree()
+        self._refresh_ops_tree_async()
+        self._configure_query_highlighting()
+        self._set_pause_visuals()
+        self.set_interval(0.5, self._refresh_status)
+
+    def _configure_query_highlighting(self) -> None:
+        query_input = self.query_one("#query_input", TextArea)
+        query_input.language = "javascript"
+        dsl_theme = TextAreaTheme(
+            "dsqa_ide",
+            base_style=Style.parse("#d7deee on #171a22"),
+            gutter_style=Style.parse("#8a93a8 on #1c212b"),
+            cursor_style=Style.parse("#ffffff on #2b3242"),
+            cursor_line_style=Style.parse("on #202534"),
+            cursor_line_gutter_style=Style.parse("#a6afc4 on #202534"),
+            selection_style=Style.parse("on #2a3f67"),
+            bracket_matching_style=Style.parse("bold #7fd9ff"),
+            syntax_styles={
+                "comment": Style.parse("italic #6f778a"),
+                "variable": Style.parse("bold #4ec9b0"),
+                "function": Style.parse("bold #4fc1ff"),
+                "function.method": Style.parse("bold #70d4ff"),
+                "parameter": Style.parse("#c6d4f8"),
+                "string": Style.parse("#d7ba7d"),
+                "number": Style.parse("#b5cea8"),
+                "constant.builtin": Style.parse("bold #569cd6"),
+            },
+        )
+        query_input.register_theme(dsl_theme)
+        query_input.theme = "dsqa_ide"
+        try:
+            query_input.update_highlight_query(self.DSL_HIGHLIGHT_QUERY)
+        except Exception:
+            pass
+
+    def on_shutdown(self) -> None:
+        stop_interop()
+
+    @staticmethod
+    def _ensure_interop_started() -> None:
+        try:
+            start_interop()
+        except Exception:
+            pass
+
+    def action_execute(self) -> None:
+        tabs = self.query_one("#results_tabs", TabbedContent)
+        tabs.active = "data_tab"
+        self._grid_updates_enabled = True
+        self._set_stream_toggle_label()
+        self._set_pause_visuals()
+        self._execute_query_async()
+
+    def action_noop(self) -> None:
+        pass
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        button_id = event.button.id
+        if button_id == "search_button":
+            self._search_results()
+        elif button_id == "copy_button":
+            self._copy_selected_cell()
+        elif button_id == "export_button":
+            self._export_results_csv()
+        elif button_id == "execute_button":
+            self.action_execute()
+        elif button_id == "window_close_button":
+            self.exit()
+        elif button_id == "stream_toggle_button":
+            self._toggle_stream_painting()
+
+    def on_input_submitted(self, event: Input.Submitted) -> None:
+        if event.input.id == "results_search":
+            self._search_results()
+
+    def on_click(self, event: events.Click) -> None:
+        if getattr(event.widget, "id", None) != "ops_tree":
+            return
+        line = event.style.meta.get("line")
+        if line is None:
+            return
+        tree = self.query_one("#ops_tree", Tree)
+        node = tree.get_node_at_line(line)
+        if node is None:
+            return
+        node_data = node.data
+        if not isinstance(node_data, dict) or node_data.get("kind") != "constructor":
+            return
+        self._append_operator_to_query(self._stringify(node_data.get("op", "")).strip())
+
+    def _append_operator_to_query(self, op_name: str) -> None:
+        if not op_name:
+            return
+        query_input = self.query_one("#query_input", TextArea)
+        existing = query_input.text
+        if existing.strip():
+            separator = "" if existing.endswith("\n") else "\n"
+            query_input.text = f"{existing}{separator}{op_name}"
+        else:
+            query_input.text = op_name
+        query_input.focus()
+
+    def _execute_query_async(self) -> None:
+        self._ensure_interop_started()
+        query = self.query_one(TextArea).text.strip()
+        if not query:
+            return
+        self._column_width_cache.clear()
+        self._save_last_query(query)
+        self._update_timestamps.clear()
+        self._last_update_at = None
+        self._stream_started_at = time.monotonic()
+        self._total_updates = 0
+        self._last_row_count = 0
+        self._paused_latest_response = None
+        self._refresh_status()
+
+        req_id = str(uuid.uuid4())
+
+        def _run() -> None:
+            try:
+                def _on_response(resp: DslResponse) -> None:
+                    self.call_from_thread(self._render_response, resp)
+
+                get_dsl(query, req_id, _on_response)
+            except Exception as exc:
+                error_resp = DslResponse(data_packet=None, req_id=req_id, error=str(exc))
+                self.call_from_thread(self._render_response, error_resp)
+
+        threading.Thread(target=_run, daemon=True).start()
+
+    def _render_response(self, response: DslResponse) -> None:
+        if not self._grid_updates_enabled:
+            self._paused_latest_response = response
+            return
+        self._render_response_internal(response)
+
+    def _render_response_internal(self, response: DslResponse) -> None:
+        self._record_update()
+        table = self.query_one(DataTable)
+        table.remove_class("error")
+        prev_scroll_x = table.scroll_offset.x
+        prev_scroll_y = table.scroll_offset.y
+        table.clear(columns=True)
+
+        if response.error:
+            table.add_class("error")
+            error_text = self._stringify(response.error)
+            error_lines = error_text.splitlines() or [error_text]
+            self._current_col_names = []
+            self._current_rows = []
+            self._last_row_count = len(error_lines)
+            self._refresh_ops_tree()
+            table.add_columns("error")
+            for line in error_lines:
+                table.add_row(Text(line, style="bold #f17486"))
+            return
+
+        if response.data_packet is None or not response.data_packet.state:
+            self._current_col_names = []
+            self._current_rows = []
+            self._last_row_count = 0
+            self._refresh_ops_tree()
+            table.add_columns("empty")
+            table.add_row("no data")
+            return
+
+        data_packet = response.data_packet.state[0]
+        columns = data_packet.get("columns", [])
+        rows = data_packet.get("all", [])
+        if not isinstance(rows, list):
+            rows = []
+
+        col_names = [c.get("name", "") for c in columns]
+        if not col_names:
+            col_names = list(rows[0].keys()) if rows else ["empty"]
+
+        self._current_col_names = col_names
+        self._current_rows = rows
+        self._last_row_count = len(rows)
+        self._refresh_ops_tree()
+        for col_name in col_names:
+            width = self._column_width_cache.get(col_name, len(col_name))
+            for row in rows:
+                if not isinstance(row, dict):
+                    continue
+                cell_text = self._stringify(row.get(col_name, ""))
+                width = max(width, len(cell_text))
+            self._column_width_cache[col_name] = width
+            table.add_column(col_name, width=width)
+        for row in rows:
+            table.add_row(*[self._format_cell(c, row.get(c, "")) for c in col_names])
+        # Preserve user's viewport during streaming refreshes.
+        table.scroll_to(
+            x=min(prev_scroll_x, table.max_scroll_x),
+            y=min(prev_scroll_y, table.max_scroll_y),
+            animate=False,
+        )
+
+    @staticmethod
+    def _stringify(value: Any) -> str:
+        return "" if value is None else str(value)
+
+    @staticmethod
+    def _format_cell(column_name: str, value: Any) -> str | Text:
+        text = DSQALiveConsole._stringify(value)
+        lowered = column_name.lower()
+        if lowered == "id":
+            return Text(text, style="bold #9eb1d8")
+        if lowered == "symbol":
+            return Text(text, style="bold #dce6ff")
+        if lowered == "price":
+            return Text(text, style="bold #d4ddf2")
+        if lowered == "time":
+            return Text(text, style="#aab5d0")
+        if lowered == "change":
+            cleaned = text.strip()
+            if cleaned.startswith("-"):
+                return Text(text, style="bold #f17486")
+            if cleaned.startswith("+"):
+                return Text(text, style="bold #92d67e")
+            return Text(text, style="bold #c7d3ef")
+        if lowered == "bid":
+            return Text(text, style="bold #8cf3c0 on #123526")
+        if lowered == "ask":
+            return Text(text, style="bold #ffd5d9 on #3b1b25")
+        return text
+
+    def _record_update(self) -> None:
+        now = time.monotonic()
+        self._update_timestamps.append(now)
+        self._total_updates += 1
+        if self._stream_started_at is None:
+            self._stream_started_at = now
+        self._last_update_at = datetime.now()
+        self._refresh_status()
+
+    def _refresh_status(self) -> None:
+        self._refresh_interop_heartbeat()
+        now = time.monotonic()
+        cutoff = now - 1.0
+        while self._update_timestamps and self._update_timestamps[0] < cutoff:
+            self._update_timestamps.popleft()
+
+        rate = float(len(self._update_timestamps))
+        if self._stream_started_at is not None:
+            uptime_seconds = max(0, int(now - self._stream_started_at))
+            uptime = self._format_duration(uptime_seconds)
+        else:
+            uptime = "--:--:--"
+        last_update = self._last_update_at.strftime("%H:%M:%S") if self._last_update_at else "--:--:--"
+        status = Text()
+        if self._interop_is_running:
+            status.append("Interop: ● UP", style="bold #92d67e")
+        elif self._interop_exit_code is not None:
+            status.append(f"Interop: ● DOWN({self._interop_exit_code})", style="bold #f17486")
+        else:
+            status.append("Interop: ● DOWN", style="bold #8b90a1")
+        status.append(" | ")
+        status.append(f"Rows: {self._last_row_count} | Rate: {rate:.1f}/s | ")
+        status.append(f"Total Updates: {self._total_updates}", style="bold #79afe7")
+        status.append(f" | Uptime: {uptime} | ")
+        status.append(f"Last Update: {last_update}", style="bold #79afe7")
+        self.query_one("#live_status", Static).update(status)
+
+    def _refresh_interop_heartbeat(self) -> None:
+        is_running, exit_code = get_interop_status()
+        self._interop_is_running = is_running
+        self._interop_exit_code = exit_code
+
+    @staticmethod
+    def _format_duration(total_seconds: int) -> str:
+        hours = total_seconds // 3600
+        minutes = (total_seconds % 3600) // 60
+        seconds = total_seconds % 60
+        return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+    def _search_results(self) -> None:
+        raw = self.query_one("#results_search", Input).value.strip()
+        if ":" not in raw:
+            return
+        col_part, value_part = raw.split(":", 1)
+        target_col = col_part.strip()
+        target_value = value_part.strip()
+        if not target_col:
+            return
+
+        lookup = {name.lower(): idx for idx, name in enumerate(self._current_col_names)}
+        col_idx = lookup.get(target_col.lower())
+        if col_idx is None:
+            return
+
+        col_name = self._current_col_names[col_idx]
+        needle = target_value.lower()
+        for row_idx, row in enumerate(self._current_rows):
+            if not isinstance(row, dict):
+                continue
+            cell_value = self._stringify(row.get(col_name, ""))
+            if needle in cell_value.lower():
+                table = self.query_one("#results_table", DataTable)
+                table.move_cursor(row=row_idx, column=col_idx)
+                table.focus()
+                return
+
+    def _copy_selected_cell(self) -> None:
+        table = self.query_one("#results_table", DataTable)
+        if table.row_count == 0 or not self._current_col_names:
+            return
+        coordinate = table.cursor_coordinate
+        try:
+            cell = table.get_cell_at(coordinate)
+        except Exception:
+            return
+        if isinstance(cell, Text):
+            value = cell.plain
+        else:
+            value = self._stringify(cell)
+        self.copy_to_clipboard(value)
+
+    def _export_results_csv(self) -> None:
+        if not self._current_col_names:
+            return
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        export_path = os.path.join(_here, f"DSQA_RESULTS_EXPORT_{timestamp}.csv")
+        with open(export_path, "w", newline="", encoding="utf-8") as csv_file:
+            writer = csv.writer(csv_file)
+            writer.writerow(self._current_col_names)
+            for row in self._current_rows:
+                if isinstance(row, dict):
+                    writer.writerow([self._stringify(row.get(col, "")) for col in self._current_col_names])
+
+    def _refresh_ops_tree(self) -> None:
+        tree = self.query_one("#ops_tree", Tree)
+        root = tree.root
+        root.remove_children()
+
+        if self._ops_catalog_error:
+            root.add(Text(self._ops_catalog_error, style="bold #f17486"))
+            root.expand()
+            return
+
+        grouped: dict[str, dict[str, dict[str, set[str]]]] = {}
+        for row in self._ops_catalog_rows:
+            if not isinstance(row, dict):
+                continue
+            row_lookup = {self._stringify(k).lower(): v for k, v in row.items()}
+            assembly_name = self._stringify(row_lookup.get("assembly_name", "")).strip() or "(unknown assembly)"
+            namespace = self._stringify(row_lookup.get("namespace", "")).strip() or "(global)"
+            op = self._stringify(row_lookup.get("op", "")).strip() or "(unknown op)"
+            constructor = self._stringify(row_lookup.get("constructor", "")).strip() or "(none)"
+            grouped.setdefault(assembly_name, {}).setdefault(namespace, {}).setdefault(op, set()).add(constructor)
+
+        if not grouped:
+            root.add(Text("No operator metadata available", style="#8ea0c6"))
+            root.expand()
+            return
+
+        for assembly_name in sorted(grouped.keys(), key=str.lower):
+            assembly_node = root.add(Text(assembly_name, style="bold #79afe7"))
+            for namespace in sorted(grouped[assembly_name].keys(), key=str.lower):
+                namespace_node = assembly_node.add(Text(namespace, style="#cfd7eb"))
+                for op in sorted(grouped[assembly_name][namespace].keys(), key=str.lower):
+                    op_node = namespace_node.add(
+                        Text(op, style="#b8c8e7"),
+                        data={"kind": "op", "op": op},
+                    )
+                    for constructor in sorted(grouped[assembly_name][namespace][op], key=str.lower):
+                        op_node.add(
+                            Text(constructor, style="#9eb0d4"),
+                            data={"kind": "constructor", "op": op, "constructor": constructor},
+                        )
+                    op_node.expand()
+                namespace_node.expand()
+            assembly_node.expand()
+        root.expand()
+
+    def _refresh_ops_tree_async(self) -> None:
+        if self._ops_catalog_loaded or self._ops_catalog_loading:
+            return
+        self._ensure_interop_started()
+        self._ops_catalog_loading = True
+        req_id = str(uuid.uuid4())
+
+        def _run() -> None:
+            try:
+                def _on_response(resp: DslResponse) -> None:
+                    self.call_from_thread(self._update_ops_catalog, resp)
+
+                get_dsl("OpsStream()", req_id, _on_response)
+            except Exception as exc:
+                error_resp = DslResponse(data_packet=None, req_id=req_id, error=str(exc))
+                self.call_from_thread(self._update_ops_catalog, error_resp)
+
+        threading.Thread(target=_run, daemon=True).start()
+
+    def _update_ops_catalog(self, response: DslResponse) -> None:
+        self._ops_catalog_loading = False
+        if response.error:
+            self._ops_catalog_rows = []
+            self._ops_catalog_error = self._stringify(response.error)
+            self._refresh_ops_tree()
+            return
+
+        if response.data_packet is None or not response.data_packet.state:
+            self._ops_catalog_rows = []
+            self._ops_catalog_error = "OpsStream() returned no data"
+            self._refresh_ops_tree()
+            return
+
+        data_packet = response.data_packet.state[0]
+        rows = data_packet.get("all", [])
+        if not isinstance(rows, list):
+            rows = []
+        normalized_rows: list[dict[str, Any]] = []
+        for row in rows:
+            if isinstance(row, dict):
+                normalized_rows.append(row)
+        self._ops_catalog_rows = normalized_rows
+        self._ops_catalog_error = None
+        self._ops_catalog_loaded = True
+        self._refresh_ops_tree()
+
+    def _toggle_stream_painting(self) -> None:
+        self._grid_updates_enabled = not self._grid_updates_enabled
+        self._set_stream_toggle_label()
+        self._set_pause_visuals()
+        if self._grid_updates_enabled and self._paused_latest_response is not None:
+            latest = self._paused_latest_response
+            self._paused_latest_response = None
+            self._render_response_internal(latest)
+
+    def _set_stream_toggle_label(self) -> None:
+        button = self.query_one("#stream_toggle_button", Button)
+        button.label = "⏸ Pause" if self._grid_updates_enabled else "▶ Play"
+
+    def _set_pause_visuals(self) -> None:
+        table = self.query_one("#results_table", DataTable)
+        panel = self.query_one("#results_panel", Vertical)
+        if self._grid_updates_enabled:
+            table.remove_class("paused")
+            panel.remove_class("paused")
+        else:
+            table.add_class("paused")
+            panel.add_class("paused")
+
+    def _load_last_query(self) -> str:
+        try:
+            with open(self.LAST_QUERY_PATH, "r", encoding="utf-8") as file:
+                text = file.read().strip()
+                return text if text else self.DEFAULT_QUERY
+        except FileNotFoundError:
+            return self.DEFAULT_QUERY
+        except OSError:
+            return self.DEFAULT_QUERY
+
+    def _save_last_query(self, query: str) -> None:
+        try:
+            with open(self.LAST_QUERY_PATH, "w", encoding="utf-8") as file:
+                file.write(query)
+        except OSError:
+            pass
+
+
+if __name__ == "__main__":
+    DSQALiveConsole().run()

src/DataStreamSuite/BlueOrbDSLInteropServer/BlueOrbDSLInteropServer.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AssemblyName>BlueOrbDSLInteropServer</AssemblyName>
+    <RootNamespace>BlueOrbDSLInteropServer</RootNamespace>
+  </PropertyGroup>
+
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\..\cs\Bts.Spark.Api\Bts.Spark.Api.csproj" />
+        <ProjectReference Include="..\DataStream\DataStream.csproj" />
+        <ProjectReference Include="..\PubSub\PubSub.csproj" />
+        <ProjectReference Include="..\BlueOrbOperatorStreams\BlueOrbOperatorStreams.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="Bts.Spark.Api" Version="$(SparkVersion)" />
+        <PackageReference Include="DataStream" Version="$(SparkVersion)" />
+        <PackageReference Include="PubSub" Version="$(SparkVersion)" />
+        <PackageReference Include="BlueOrbOperatorStreams" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
+
+</Project>

src/DataStreamSuite/BlueOrbDSLInteropServer/Program.cs

@@ -0,0 +1,122 @@
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+using BCG;
+
+using bts.BlueOrb;
+
+using Bts.DataStream;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+using Bts.PubSub;
+
+using MathNet.Numerics.Statistics;
+
+
+namespace BlueOrbDSLInteropServer {
+   // BlueOrbDSLInteropServer reads DSL lines from stdin, validates them,
+   // and writes one DataPacket JSON line to stdout per input.
+   internal static class Program {
+      private const string ExitSignal = ":exit";
+      private static SyncQueue<DslRequest> _work_queue = new SyncQueue<DslRequest>();
+      private static JsonSerializerOptions response_json_options = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      private static string _current_req_id;
+
+      // Request from Python
+      private sealed class DslRequest {
+         public string? dsl { get; set; }
+         public string? req_id { get; set; }
+      }
+
+      // Response envelope for Python: includes DataPacket, request id, and optional error.
+      private sealed class DslResponse {
+         public DataPacket? data_packet { get; set; }
+         public string? req_id { get; set; }
+         public string? error { get; set; }
+      }
+
+      private static void Main(string[] args) {
+         //System.Threading.Thread.Sleep(20000);
+         System.Threading.Thread thread = new Thread(_worker_loop);
+         thread.IsBackground = true;
+         thread.Start();
+         response_json_options.Converters.Add(new DataPacketJsonConverter {
+            IncludeCurrent = true,
+            UseObjectRows = true,
+         });
+         string? line;
+         while ((line = Console.ReadLine()) != null) {
+            line = line.Trim();
+            if (line.Length == 0) {
+               continue;
+            }
+            if (string.Equals(line, ExitSignal, StringComparison.OrdinalIgnoreCase)) {
+               break;
+            }
+
+            try {
+               DslRequest req = JsonSerializer.Deserialize<DslRequest>(line);
+               _work_queue.enqueue(req);
+            } catch (Exception ex) {
+               bts.Log.assert(false, ex.ToString());
+            }
+         }
+      }
+
+      private static void _worker_loop() {
+         Dictionary<string, ViewStream> req_map = new Dictionary<string, ViewStream>();
+         while (true) {
+            System.Threading.Thread.Sleep(100);
+            ViewStreamDatabase.instance.doWork();
+            var new_reqs = _work_queue.dequeue_all();
+            if (new_reqs == null || new_reqs.Count == 0) {
+               continue;
+            }
+            // ubsubscribing to previous stream(s)
+            foreach (var kv in req_map) {
+               kv.Value.onChange -= dp_obj => _on_dp(kv.Key, dp_obj);
+            }
+            foreach (var r in new_reqs) {
+               var req_id = string.IsNullOrWhiteSpace(r.req_id) ? Guid.NewGuid().ToString() : r.req_id;
+               _current_req_id = req_id;
+               try {
+                  DSLStream dsl_str = new DSLStream(r.dsl);
+                  req_map[req_id] = dsl_str;
+                  dsl_str.onChange += dp_obj => _on_dp(req_id, dp_obj);
+               } catch (Exception ex) {
+                  _write_error(req_id, ex.StackTrace);
+               }
+            }
+         }
+      }
+
+      private static void _on_dp(string req_id, object dp_obj) {
+         // eventhough we unsubscribe to all the other streams except the currrent one, 
+         // sometimes the onLastSub is not implemented and we still get updates, this masks them 
+         if (req_id != _current_req_id)
+            return;
+         DataPacket dp = dp_obj as DataPacket;
+         DslResponse response = new DslResponse() {
+            data_packet = dp,
+            req_id = req_id,
+            error = ""
+         };
+         var json = JsonSerializer.Serialize(response, response_json_options);
+         Console.WriteLine(json);
+      }
+
+      private static void _write_error(string req_id, string error) {
+         DslResponse response = new DslResponse() {
+            data_packet = null,
+            req_id = req_id,
+            error = error
+         };
+         var json = JsonSerializer.Serialize(response, response_json_options);
+         Console.WriteLine(json);
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/BarBuilderStream_depleted.cs

@@ -0,0 +1,514 @@
+// namespace BlueOrb.Operators.Signal;
+
+// using BCG;
+
+// using bts;
+
+// using System;
+// using System.Collections.Generic;
+// using System.Globalization;
+// using System.Linq;
+
+// [DataStreamUserRespresentation("BarBuilderStream")]
+// public class BarBuilderStream : ViewStream
+// {
+//    enum BarType
+//    {
+//       TIME,
+//       TICK,
+//       VOLUME,
+//       DOLLAR,
+//    }
+
+//    enum PriceSource
+//    {
+//       trade_price,
+//       mid,
+//       bid,
+//       ask,
+//    }
+
+//    enum VolumeSource
+//    {
+//       trade_size,
+//       none,
+//    }
+
+//    enum SessionReset
+//    {
+//       NONE,
+//       DAY,
+//       RTH,
+//       ETH,
+//    }
+
+//    sealed class BarState
+//    {
+//       public string Contract = string.Empty;
+//       public DateTime StartTs;
+//       public DateTime EndTs;
+//       public double Open;
+//       public double High;
+//       public double Low;
+//       public double Close;
+//       public double Volume;
+//       public int TradeCount;
+//       public double SumPriceSize;
+//       public double SumSize;
+//       public double DollarNotional;
+//       public bool HasData;
+//       public DateTime SessionDay;
+//       public bool InitialSnapshotEmitted;
+//    }
+
+//    readonly ViewStream _priceStream;
+//    readonly BarType _barType;
+//    readonly int _barSize;
+//    readonly PriceSource _priceSource;
+//    readonly VolumeSource _volumeSource;
+//    readonly string _timestampCol;
+//    readonly bool _emitForming;
+//    readonly bool _emitInitialSnapshot;
+//    readonly SessionReset _sessionReset;
+
+//    readonly Dictionary<string, BarState> _stateByContract = new(StringComparer.Ordinal);
+//    readonly Dictionary<string, bool> _validCols = new(StringComparer.OrdinalIgnoreCase);
+//    bool _validated;
+//    int _nextId;
+//    readonly BlueTable _outTable;
+//    readonly IBlueTableSchema _outSchema;
+
+//    public BarBuilderStream(
+//       ViewStream price_stream,
+//       string bar_type,
+//       int bar_size,
+//       string price_source,
+//       string volume_source,
+//       string timestamp_col = "ts_event",
+//       int emit_forming_i = 0,
+//       string session_reset = "NONE",
+//       int emit_initial_snapshot_i = 0
+//    )
+//    {
+//       _priceStream = price_stream;
+//       _barType = ParseBarType(bar_type);
+//       _barSize = bar_size;
+//       _priceSource = ParsePriceSource(price_source);
+//       _volumeSource = ParseVolumeSource(volume_source);
+//       _timestampCol = timestamp_col;
+//       _emitForming = emit_forming_i == 1;
+//       _emitInitialSnapshot = emit_initial_snapshot_i == 1;
+//       _sessionReset = ParseSessionReset(session_reset);
+
+//       if (_barSize <= 0)
+//       {
+//          throw new ArgumentException("bar_size must be > 0");
+//       }
+
+//       _outSchema = BlueDataUtility.form_schema(
+//          "BarBuilder",
+//          Tuple.Create("id", TypeCode.Int32),
+//          Tuple.Create("ts_event", TypeCode.String),
+//          Tuple.Create("bar_start_ts", TypeCode.String),
+//          Tuple.Create("bar_end_ts", TypeCode.String),
+//          Tuple.Create("contract", TypeCode.String),
+//          Tuple.Create("open", TypeCode.Double),
+//          Tuple.Create("high", TypeCode.Double),
+//          Tuple.Create("low", TypeCode.Double),
+//          Tuple.Create("close", TypeCode.Double),
+//          Tuple.Create("volume", TypeCode.Double),
+//          Tuple.Create("trade_count", TypeCode.Int32),
+//          Tuple.Create("vwap", TypeCode.Double),
+//          Tuple.Create("is_forming", TypeCode.Boolean)
+//       );
+//       _outTable = new BlueTable(_outSchema);
+//    }
+
+//    protected override void myOnFirstSubscriber()
+//    {
+//       _priceStream.onChange += _on_price_change;
+//    }
+
+//    protected override void myOnLastSubscriber()
+//    {
+//       _priceStream.onChange -= _on_price_change;
+//    }
+
+//    void _on_price_change(object newValue)
+//    {
+//       var dp = newValue as IDataPacket;
+//       if (dp is null)
+//       {
+//          return;
+//       }
+//       if (!_validated)
+//       {
+//          ValidateInputSchema(dp.newRows.schema);
+//          _validated = true;
+//       }
+
+//       var emitted = new List<IBlueRow>();
+//       foreach (IBlueRow row in dp.addedRows)
+//       {
+//          ProcessEvent(row, emitted);
+//       }
+
+//       if (emitted.Count == 0)
+//       {
+//          return;
+//       }
+//       foreach (var r in emitted)
+//       {
+//          _outTable.addRow(r);
+//       }
+//       var outDp = BlueDataUtility.build_dp_inc(_outTable, myLastValue as IDataPacket);
+//       myFireEvent(outDp);
+//    }
+
+//    void ValidateInputSchema(IBlueTableSchema schema)
+//    {
+//       _validCols.Clear();
+//       foreach (var name in schema.columnNames)
+//       {
+//          _validCols[name] = true;
+//       }
+
+//       if (!_validCols.ContainsKey(_timestampCol))
+//       {
+//          throw new ArgumentException($"Missing required column: {_timestampCol}");
+//       }
+
+//       switch (_priceSource)
+//       {
+//          case PriceSource.trade_price:
+//             RequireCol("trade_price");
+//             break;
+//          case PriceSource.mid:
+//             RequireCol("bid_price");
+//             RequireCol("ask_price");
+//             break;
+//          case PriceSource.bid:
+//             RequireCol("bid_price");
+//             break;
+//          case PriceSource.ask:
+//             RequireCol("ask_price");
+//             break;
+//       }
+
+//       if (_barType is BarType.VOLUME or BarType.DOLLAR)
+//       {
+//          if (_volumeSource == VolumeSource.none || !_validCols.ContainsKey("trade_size"))
+//          {
+//             throw new ArgumentException("VOLUME/DOLLAR require executed trade volume; invalid for BBO-only streams.");
+//          }
+//       }
+//       else if (_volumeSource == VolumeSource.trade_size)
+//       {
+//          RequireCol("trade_size");
+//       }
+//    }
+
+//    void RequireCol(string name)
+//    {
+//       if (!_validCols.ContainsKey(name))
+//       {
+//          throw new ArgumentException($"Missing required column for selected source: {name}");
+//       }
+//    }
+
+//    void ProcessEvent(IBlueRow row, List<IBlueRow> emitted)
+//    {
+//       var ts = ParseTimestamp(row);
+//       var contract = ResolveContract(row);
+//       var price = ResolvePrice(row);
+//       var size = ResolveSize(row);
+//       var state = GetOrCreateState(contract);
+
+//       if (state.HasData && _sessionReset != SessionReset.NONE && ts.Date != state.SessionDay.Date)
+//       {
+//          emitted.Add(ToBarRow(state, ts_event: state.EndTs, isForming: false));
+//          state.HasData = false;
+//       }
+
+//       switch (_barType)
+//       {
+//          case BarType.TIME:
+//             ProcessTimeBar(state, ts, price, size, emitted);
+//             break;
+//          case BarType.TICK:
+//             ProcessThresholdBar(state, ts, price, size, emitted, state.TradeCount + 1 >= _barSize);
+//             break;
+//          case BarType.VOLUME:
+//             ProcessThresholdBar(state, ts, price, size, emitted, state.Volume + size >= _barSize);
+//             break;
+//          case BarType.DOLLAR:
+//             ProcessThresholdBar(state, ts, price, size, emitted, state.DollarNotional + (price * size) >= _barSize);
+//             break;
+//       }
+//    }
+
+//    void ProcessTimeBar(BarState state, DateTime ts, double price, double size, List<IBlueRow> emitted)
+//    {
+//       var bucketStart = AlignTimeBucket(ts, _barSize);
+//       var bucketEnd = bucketStart.AddSeconds(_barSize);
+
+//       if (!state.HasData)
+//       {
+//          StartBar(state, ts, bucketStart, bucketEnd, price, size);
+//          MaybeEmitInitialSnapshot(state, ts, emitted);
+//          if (_emitForming)
+//          {
+//             emitted.Add(ToBarRow(state, ts, isForming: true));
+//          }
+//          return;
+//       }
+
+//       if (bucketStart < state.StartTs)
+//       {
+//          Log.Gen.error(
+//             $"BarBuilderStream ignored out-of-order event. contract={state.Contract} ts={ToIso(ts)} " +
+//             $"current_bar_start={ToIso(state.StartTs)}");
+//          return;
+//       }
+
+//       if (bucketStart > state.StartTs)
+//       {
+//          emitted.Add(ToBarRow(state, state.EndTs, isForming: false));
+//          StartBar(state, ts, bucketStart, bucketEnd, price, size);
+//          if (_emitForming)
+//          {
+//             emitted.Add(ToBarRow(state, ts, isForming: true));
+//          }
+//          return;
+//       }
+
+//       Accumulate(state, price, size);
+//       state.EndTs = bucketEnd;
+//       if (_emitForming)
+//       {
+//          emitted.Add(ToBarRow(state, ts, isForming: true));
+//       }
+//    }
+
+//    void ProcessThresholdBar(
+//       BarState state,
+//       DateTime ts,
+//       double price,
+//       double size,
+//       List<IBlueRow> emitted,
+//       bool shouldClose
+//    )
+//    {
+//       if (!state.HasData)
+//       {
+//          StartBar(state, ts, ts, ts, price, size);
+//          MaybeEmitInitialSnapshot(state, ts, emitted);
+//       }
+//       else
+//       {
+//          Accumulate(state, price, size);
+//       }
+//       state.EndTs = ts;
+
+//       if (_emitForming)
+//       {
+//          emitted.Add(ToBarRow(state, ts, isForming: true));
+//       }
+//       if (shouldClose)
+//       {
+//          emitted.Add(ToBarRow(state, ts, isForming: false));
+//          state.HasData = false;
+//       }
+//    }
+
+//    static DateTime AlignTimeBucket(DateTime ts, int seconds)
+//    {
+//       var utc = ts.ToUniversalTime();
+//       long bucketTicks = TimeSpan.FromSeconds(seconds).Ticks;
+//       long aligned = (utc.Ticks / bucketTicks) * bucketTicks;
+//       return new DateTime(aligned, DateTimeKind.Utc);
+//    }
+
+//    void MaybeEmitInitialSnapshot(BarState state, DateTime ts, List<IBlueRow> emitted)
+//    {
+//       if (!_emitInitialSnapshot || state.InitialSnapshotEmitted)
+//       {
+//          return;
+//       }
+//       emitted.Add(ToBarRow(state, ts, isForming: false));
+//       state.InitialSnapshotEmitted = true;
+//    }
+
+//    void StartBar(
+//       BarState state,
+//       DateTime ts,
+//       DateTime barStart,
+//       DateTime barEnd,
+//       double price,
+//       double size
+//    )
+//    {
+//       state.StartTs = barStart;
+//       state.EndTs = barEnd;
+//       state.Open = price;
+//       state.High = price;
+//       state.Low = price;
+//       state.Close = price;
+//       state.Volume = 0;
+//       state.TradeCount = 0;
+//       state.SumPriceSize = 0;
+//       state.SumSize = 0;
+//       state.DollarNotional = 0;
+//       state.SessionDay = ts.Date;
+//       state.HasData = true;
+//       Accumulate(state, price, size);
+//    }
+
+//    static void Accumulate(BarState state, double price, double size)
+//    {
+//       state.High = Math.Max(state.High, price);
+//       state.Low = Math.Min(state.Low, price);
+//       state.Close = price;
+//       state.TradeCount++;
+//       if (size > 0)
+//       {
+//          state.Volume += size;
+//          state.SumSize += size;
+//          state.SumPriceSize += price * size;
+//          state.DollarNotional += price * size;
+//       }
+//    }
+
+//    IBlueRow ToBarRow(BarState state, DateTime ts_event, bool isForming)
+//    {
+//       double vwap = state.SumSize > 0 ? state.SumPriceSize / state.SumSize : state.Close;
+//       var row = new MutableBlueRow(_outSchema);
+//       row.setValue("id", _nextId++);
+//       row.setValue("ts_event", ToIso(ts_event));
+//       row.setValue("bar_start_ts", ToIso(state.StartTs));
+//       row.setValue("bar_end_ts", ToIso(state.EndTs));
+//       row.setValue("contract", state.Contract);
+//       row.setValue("open", state.Open);
+//       row.setValue("high", state.High);
+//       row.setValue("low", state.Low);
+//       row.setValue("close", state.Close);
+//       row.setValue("volume", _volumeSource == VolumeSource.none ? 0.0 : state.Volume);
+//       row.setValue("trade_count", state.TradeCount);
+//       row.setValue("vwap", vwap);
+//       row.setValue("is_forming", isForming);
+//       return row;
+//    }
+
+//    BarState GetOrCreateState(string contract)
+//    {
+//       if (_stateByContract.TryGetValue(contract, out var existing))
+//       {
+//          return existing;
+//       }
+//       var created = new BarState { Contract = contract };
+//       _stateByContract[contract] = created;
+//       return created;
+//    }
+
+//    DateTime ParseTimestamp(IBlueRow row)
+//    {
+//       object? raw = row[_timestampCol];
+//       if (raw is DateTime dt)
+//       {
+//          return dt;
+//       }
+//       if (raw is string s && DateTime.TryParse(s, null, DateTimeStyles.RoundtripKind, out dt))
+//       {
+//          return dt;
+//       }
+//       throw new ArgumentException($"Invalid timestamp value in column '{_timestampCol}'.");
+//    }
+
+//    string ResolveContract(IBlueRow row)
+//    {
+//       if (_validCols.ContainsKey("contract"))
+//       {
+//          return (row["contract"]?.ToString() ?? string.Empty).Trim();
+//       }
+//       if (_validCols.ContainsKey("symbol"))
+//       {
+//          return (row["symbol"]?.ToString() ?? string.Empty).Trim();
+//       }
+//       return string.Empty;
+//    }
+
+//    double ResolvePrice(IBlueRow row)
+//    {
+//       return _priceSource switch
+//       {
+//          PriceSource.trade_price => ToDouble(row["trade_price"], "trade_price"),
+//          PriceSource.mid => (ToDouble(row["bid_price"], "bid_price") + ToDouble(row["ask_price"], "ask_price")) / 2.0,
+//          PriceSource.bid => ToDouble(row["bid_price"], "bid_price"),
+//          PriceSource.ask => ToDouble(row["ask_price"], "ask_price"),
+//          _ => throw new ArgumentOutOfRangeException()
+//       };
+//    }
+
+//    double ResolveSize(IBlueRow row)
+//    {
+//       if (_volumeSource == VolumeSource.none)
+//       {
+//          return 0;
+//       }
+//       return ToDouble(row["trade_size"], "trade_size");
+//    }
+
+//    static double ToDouble(object? value, string colName)
+//    {
+//       var d = BlueDataUtility.GetDouble(value!);
+//       if (!d.HasValue)
+//       {
+//          throw new ArgumentException($"Column '{colName}' must be numeric.");
+//       }
+//       return d.Value;
+//    }
+
+//    static string ToIso(DateTime dt) =>
+//       dt.ToUniversalTime().ToString("O");
+
+//    static BarType ParseBarType(string value)
+//    {
+//       if (Enum.TryParse<BarType>(value, true, out var parsed))
+//       {
+//          return parsed;
+//       }
+//       throw new ArgumentException("bar_type must be one of: TIME | TICK | VOLUME | DOLLAR");
+//    }
+
+//    static PriceSource ParsePriceSource(string value)
+//    {
+//       if (Enum.TryParse<PriceSource>(value, true, out var parsed))
+//       {
+//          return parsed;
+//       }
+//       throw new ArgumentException("price_source must be one of: trade_price | mid | bid | ask");
+//    }
+
+//    static VolumeSource ParseVolumeSource(string value)
+//    {
+//       if (Enum.TryParse<VolumeSource>(value, true, out var parsed))
+//       {
+//          return parsed;
+//       }
+//       throw new ArgumentException("volume_source must be one of: trade_size | none");
+//    }
+
+//    static SessionReset ParseSessionReset(string value)
+//    {
+//       if (!Enum.TryParse<SessionReset>(value, true, out var parsed))
+//       {
+//          throw new ArgumentException("session_reset must be one of: NONE | DAY | RTH | ETH");
+//       }
+//       // TODO: differentiate RTH/ETH session boundaries; currently treated as DAY.
+//       if (parsed is SessionReset.RTH or SessionReset.ETH)
+//       {
+//          return SessionReset.DAY;
+//       }
+//       return parsed;
+//    }
+// }

src/DataStreamSuite/BlueOrbOperatorStreams/BlueOrbOperatorStreams.csproj

@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
+    <PackageReference Include="OpenAI" Version="2.0.0" />
+    <PackageReference Include="System.Linq.Async" Version="6.0.1" />
+  </ItemGroup>
+
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\..\cs\Bts.Spark.Api\Bts.Spark.Api.csproj" />
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+        <ProjectReference Include="..\DataStream\DataStream.csproj" />
+        <ProjectReference Include="..\PubSub\PubSub.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="Bts.Spark.Api" Version="$(SparkVersion)" />
+        <PackageReference Include="DataStream" Version="$(SparkVersion)" />
+        <PackageReference Include="PubSub" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
+
+  <!-- <ItemGroup>
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubHeartbeatSchema.json" TypeName="PubSubHeartbeatRow" TableName="PubSubHeartbeat" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubRouteSchema.json" TypeName="PubSubRouteRow" TableName="PubSubRoute" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubServerStateSchema.json" TypeName="PubSubServerStateRow" TableName="PubSubServerState" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubServerTagStatsSchema.json" TypeName="PubSubServerTagStatsRow" TableName="PubSubServerTagStats" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\TableMetadataSchema.json" TypeName="TableMetadataRow" TableName="TableMetadata" />
+  </ItemGroup> -->
+
+</Project>

src/DataStreamSuite/BlueOrbOperatorStreams/Data/ContractUtils.cs

@@ -0,0 +1,294 @@
+namespace BlueOrb.Internal
+{
+   using bts;
+   //using bts.api;
+   //using bts.spark;
+
+   using System;
+   using System.Diagnostics.CodeAnalysis;
+
+   public static class ContractUtils
+   {
+      public record FutureTradableContract(
+         string Symbol,
+         DateTime Expiration,
+         Exchange Exchange,
+         string? ContractText);
+
+      public class FutureSpreadContract
+      {
+         public List<Tuple<double, FutureTradableContract>> legs = new(2);
+         public string Symbol
+         {
+            get
+            {
+               return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.Symbol;
+            }
+         }
+         public string? ContractText;
+      }
+
+      public record FutureContinousContract(string Symbol, Exchange Exchange);
+
+      // this is for all future conytracts belonging to a symbol
+      public record FutureWildcardContract(string Symbol, Exchange Exchange);
+
+      public class FutureContract
+      {
+         public FutureWildcardContract? wildcard_contract;
+         public FutureTradableContract? future_tradable_contract;
+         public FutureSpreadContract? spread_tradable_contract;
+         public FutureContinousContract? continous_constract;
+
+         public FutureContract(string txt)
+         {
+            wildcard_contract = parse_wildcard_contract(txt);
+            if (wildcard_contract != null)
+               return;
+            var fc = parse_future_contract(txt);
+            if (fc != null)
+            {
+               future_tradable_contract = fc;
+               return;
+            }
+            var sc = parse_future_spread_contract(txt);
+            if (sc != null)
+            {
+               spread_tradable_contract = sc;
+               return;
+            }
+            // this is not totally true, since ParseFutureSpreadContract does not
+            // parse ICS spreads
+            continous_constract = new(txt, Exchange.CME);
+         }
+
+         [MemberNotNullWhen(true, nameof(continous_constract))]
+         public bool is_continous_contract => continous_constract != null;
+
+         [MemberNotNullWhen(true, nameof(wildcard_contract))]
+         public bool is_wildcard_contract => wildcard_contract != null;
+
+         [MemberNotNullWhen(true, nameof(future_tradable_contract))]
+         public bool is_future_tradable_contract => future_tradable_contract != null;
+
+         [MemberNotNullWhen(true, nameof(spread_tradable_contract))]
+         public bool is_spread_tradable_contract => spread_tradable_contract != null;
+
+         public string symbol()
+         {
+            if (wildcard_contract != null)
+               return wildcard_contract.Symbol;
+            if (continous_constract != null)
+            {
+               return continous_constract.Symbol;
+            }
+            else if (future_tradable_contract != null)
+            {
+               return future_tradable_contract.Symbol;
+            }
+            else if (spread_tradable_contract != null)
+            {
+               return spread_tradable_contract.Symbol;
+            }
+            Log.assert(false);
+            return string.Empty;
+         }
+         public string? contract_text
+         {
+            get
+            {
+               if (future_tradable_contract != null)
+                  return future_tradable_contract.ContractText;
+               if (spread_tradable_contract != null)
+                  return spread_tradable_contract.ContractText;
+               return null;
+            }
+         }
+      }
+      #region Parsing
+      public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME)
+      {
+         if (contract_txt.EndsWith("*"))
+         {
+            contract_txt = contract_txt.Replace("*", string.Empty);
+            return new FutureWildcardContract(contract_txt, exchange);
+         }
+         return null;
+      }
+      public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME)
+      {
+         // Validate the future string length
+         if (!contract_txt.All(c => char.IsLetterOrDigit(c)))
+            return null;
+         if (contract_txt.Length < 4)
+         {
+            return null;
+         }
+
+         // Extract the symbol and the expiration month/year codes
+         string symbol = contract_txt.Substring(0, contract_txt.Length - 2);
+         char monthCode = contract_txt[contract_txt.Length - 2];
+         char yearCode = contract_txt[contract_txt.Length - 1];
+
+         // Map month code to month, return null if the month code is invalid
+         int month = monthCode switch
+         {
+            'F' => 1,  // January
+            'G' => 2,  // February
+            'H' => 3,  // March
+            'J' => 4,  // April
+            'K' => 5,  // May
+            'M' => 6,  // June
+            'N' => 7,  // July
+            'Q' => 8,  // August
+            'U' => 9,  // September
+            'V' => 10, // October
+            'X' => 11, // November
+            'Z' => 12, // December
+            _ => 0 // Return 0 for invalid month code
+         };
+
+         if (month == 0)
+         {
+            return null; // Invalid month code
+         }
+
+         // Calculate the year based on the current decade
+         int currentYear = DateTime.Now.Year;
+         int decadeStart = currentYear / 10 * 10;
+         int year = decadeStart + (yearCode - '0');
+
+         // If the year is in the past, assume the next decade
+         if (year < currentYear)
+         {
+            year += 10;
+         }
+
+         // Create the FutureContract instance and set its properties
+         DateTime expiration = new DateTime(year, month, 1);
+         return new(symbol, expiration, exchange, contract_txt);
+      }
+      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
+      public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME)
+      {
+         // Split the string by '-' to handle each leg
+         var spreadParts = contract_txt.Split('-');
+         if (spreadParts.Length < 2)
+         {
+            return null; // Not a valid spread format
+         }
+
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };
+
+         foreach (var part in spreadParts)
+         {
+            // Split each leg by '*', assuming the format is weight*future or just future
+            string[] legParts = part.Split('*');
+            double weight = 1.0;
+            string futurePart;
+
+            if (legParts.Length == 2)
+            {
+               if (!double.TryParse(legParts[0], out weight))
+               {
+                  return null; // Invalid weight, parsing failed
+               }
+               futurePart = legParts[1];
+            }
+            else if (legParts.Length == 1)
+            {
+               futurePart = legParts[0];
+            }
+            else
+            {
+               return null; // Invalid format, too many '*' parts
+            }
+
+            // Parse the future contract
+            var futureContract = parse_future_contract(futurePart, exchange);
+            if (futureContract == null)
+            {
+               return null; // Failed to parse the future part
+            }
+
+            // Add the leg to the spread contract
+            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
+         }
+         return spreadContract;
+      }
+
+      // public static ContractType DetermineFutureOrSpread(string contract) {
+      //    if (contract.Contains("-")) {
+      //       return ContractType.FutureSpread;
+      //    } else {
+      //       return ContractType.Future;
+      //    }
+      // }
+      #endregion
+
+      public static ContractType get_type(string contract_txt)
+      {
+         var fc = parse_future_contract(contract_txt);
+         if (fc != null)
+         {
+            return ContractType.Future;
+         }
+         var sc = parse_future_spread_contract(contract_txt);
+         if (sc != null)
+         {
+            return ContractType.FutureSpread;
+         }
+         // tehre are some non calendar spreads we cant parse
+         //Log.assert(false);
+         return ContractType.FutureSpread;
+      }
+
+      public static ContractCategory get_category(string symbol)
+      {
+         var tp = get_symbol_category(symbol);
+         return tp.Item1;
+      }
+
+      public static Tuple<ContractCategory, string> get_symbol_category(string symbol)
+      {
+         var fc = parse_future_contract(symbol);
+         if (fc != null)
+         {
+            return Tuple.Create(ContractCategory.FutureString, fc.Symbol);
+         }
+         var sc = parse_future_spread_contract(symbol);
+         if (sc != null)
+         {
+            return Tuple.Create(ContractCategory.FutureString, sc.Symbol);
+         }
+         return Tuple.Create(ContractCategory.Symbol, symbol);
+      }
+
+
+
+      public enum ContractType
+      {
+         Spot,
+         Option,
+         Future,
+         FutureSpread,
+         OptionSpread
+      }
+      public enum ContractCategory
+      {
+         Symbol = 0,
+         FutureString,
+         SpreadString
+      }
+      public enum CallPut
+      {
+         Call,
+         Put
+      }
+      public enum Exchange
+      {
+         CME
+      }
+
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Data/ReplayFutureLiveBBOStream.cs

@@ -0,0 +1,99 @@
+namespace BlueOrb.Operators.Data {
+   using BCG;
+
+   using bts;
+
+   using Spark.Core.Diagnostics;
+
+   using BlueOrb.Internal;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("ReplayFutureLiveBBOStream")]
+   public class ReplayFutureLiveBBOStream : ViewStream, BCG.Utilities.General.IWorker {
+      private class LastFireTime {
+         public DateTime last_fireitme_file = DateTime.MinValue;
+         public DateTime last_firetime_clock = DateTime.MinValue;
+      }
+      private class RowsWithPointer {
+         public List<IBlueRow>? rows;
+         public int pointer;
+      }
+
+
+      readonly ViewStreamDatabase? _vsd;
+      readonly ContractUtils.FutureContinousContract? _fcc;
+      private readonly LastFireTime _last_fire_time = new LastFireTime();
+      private RowsWithPointer _rows = new RowsWithPointer();
+
+      public ReplayFutureLiveBBOStream(string continous_contract) {
+         Func<string> iso_8601_now = () => {
+            DateTime utc_now = DateTime.UtcNow;
+            long ticks = utc_now.Ticks; // 1 tick = 100 ns
+            long nanoseconds = ticks % TimeSpan.TicksPerSecond * 100;
+            string formatted = utc_now.ToString("yyyy-MM-ddTHH:mm:ss");
+            formatted += $".{nanoseconds:D9}Z";
+            return formatted;
+         };
+         ViewStream rs = new ReplayFutureLiveBBOStream(continous_contract, true);
+         IBlueTableSchema schema_out = BlueDataUtility.form_schema("BBO",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("ts_event", TypeCode.String),
+               Tuple.Create("contract", TypeCode.String),
+               Tuple.Create("bid_price", TypeCode.Double),
+               Tuple.Create("bid_size", TypeCode.Int32),
+               Tuple.Create("ask_price", TypeCode.Double),
+               Tuple.Create("ask_size", TypeCode.Int32));
+         rs = new MapStream(rs, s => schema_out,
+            r =>
+            new BlueRow(schema_out, r["id"], /*r.get<string>("ts_event")*/iso_8601_now(), r.get<string>("symbol"),
+            r.get<double>("bid_px_00"), r.get<int>("bid_sz_00"),
+            r.get<double>("ask_px_00"), r.get<int>("ask_sz_00")));
+         mySetRealSource(rs);
+      }
+
+      private ReplayFutureLiveBBOStream(string continous_contract, bool noop)
+         : this(ViewStreamDatabase.instance, new ContractUtils.FutureContract(continous_contract)) { }
+
+      public ReplayFutureLiveBBOStream(ViewStreamDatabase vsd, ContractUtils.FutureContract fc) {
+         var fcc = fc.continous_constract;
+         Log.assert(fcc != null);
+         _vsd = vsd;
+         _fcc = fcc;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         Log.notnull(_fcc);
+         var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.Symbol}_small.csv".ToLower());
+         var rows = BCG.CSVReaderStream.LoadFile(file_path);
+         _rows.rows = new(rows.Count);
+         _rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
+         _rows.pointer = 0;
+         ExceptionAssert.NotNull(_vsd).addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         ExceptionAssert.NotNull(_vsd).removeWorker(this);
+      }
+
+      public void doWork() {
+         var rows = ExceptionAssert.NotNull(_rows.rows);
+         IBlueRow cand_row = rows[_rows.pointer];
+         string cand_ts_event = cand_row.get<string>("ts_event");
+         DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
+         TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
+         TimeSpan ts_clock = DateTime.Now - _last_fire_time.last_firetime_clock;
+         if (ts_clock >= cand_ts_span_file) {
+            _rows.pointer++;
+            _last_fire_time.last_firetime_clock = DateTime.Now;
+            _last_fire_time.last_fireitme_file = cand_ts_event_dt;
+            BlueTable bt_out = new BlueTable(cand_row.schema);
+            bt_out.addRow(cand_row);
+            var dp_out = BlueDataUtility.build_dp(bt_out, myLastValue as IDataPacket);
+            myFireEvent(dp_out);
+         }
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Data/ReplayFutureLiveTradeStream.cs

@@ -0,0 +1,111 @@
+namespace Blueorb.Operators.Data
+{
+   using BCG;
+
+   using Spark.Core.Diagnostics;
+   using BlueOrb.Internal;
+   using bts;
+   using BlueOrb.Operators.Core;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("ReplayFutureLiveTradeStream")]
+   public class ReplayFutureLiveTradeStream : ViewStream, BCG.Utilities.General.IWorker
+   {
+      private class LastFireTime
+      {
+         public DateTime last_fireitme_file = DateTime.MinValue;
+         public DateTime last_firetime_clock = DateTime.MinValue;
+      }
+      private class RowsWithPointer
+      {
+         public List<IBlueRow>? rows;
+         public int pointer;
+      }
+
+
+      readonly ViewStreamDatabase? _vsd;
+      readonly ContractUtils.FutureContinousContract? _fcc;
+      private readonly LastFireTime _last_fire_time = new LastFireTime();
+      readonly RowsWithPointer _rows = new();
+
+      public ReplayFutureLiveTradeStream(string continous_contract)
+      {
+         Func<string> iso_8601_now = () =>
+         {
+            DateTime utc_now = DateTime.UtcNow;
+            long ticks = utc_now.Ticks; // 1 tick = 100 ns
+            long nanoseconds = ticks % TimeSpan.TicksPerSecond * 100;
+            string formatted = utc_now.ToString("yyyy-MM-ddTHH:mm:ss");
+            formatted += $".{nanoseconds:D9}Z";
+            return formatted;
+         };
+         ViewStream rs = new ReplayFutureLiveTradeStream(continous_contract, true);
+         IBlueTableSchema schema_out = BlueDataUtility.form_schema("BBO",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("ts_event", TypeCode.String),
+               Tuple.Create("contract", TypeCode.String),
+               Tuple.Create("trade_size", TypeCode.Double),
+               Tuple.Create("trade_price", TypeCode.Int32),
+               Tuple.Create("aggressor_side", TypeCode.String));
+         rs = new MapStream(rs, s => schema_out,
+            r =>
+            new BlueRow(schema_out, r["id"], /*r.get<string>("ts_event")*/iso_8601_now(), r.get<string>("symbol"),
+            r.get<double>("size"), r.get<double>("price"),
+            r.get<string>("side").StartsWith("B") ? "BUY" : (r.get<string>("side").StartsWith("A") ? "SELL" : "NONE")));
+         rs = new TimeSeriesStream(rs);
+         mySetRealSource(rs);
+      }
+
+      private ReplayFutureLiveTradeStream(string continous_contract, bool noop)
+         : this(ViewStreamDatabase.instance, new ContractUtils.FutureContract(continous_contract)) { }
+
+      public ReplayFutureLiveTradeStream(ViewStreamDatabase vsd, ContractUtils.FutureContract fc)
+      {
+         var fcc = fc.continous_constract;
+         Log.assert(fcc != null);
+         _vsd = vsd;
+         _fcc = fcc;
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         Log.notnull(_vsd);
+         Log.notnull(_fcc);
+         var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.Symbol}.csv");
+         var csv_rd = new BlueOrb.Operators.Core.CSVReaderStream(file_path, false);
+         var rows = BCG.CSVReaderStream.LoadFile(file_path);
+         _rows.rows = new(rows.Count);
+         _rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
+         _rows.pointer = 0;
+         ExceptionAssert.NotNull(_vsd).addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         ExceptionAssert.NotNull(_vsd).removeWorker(this);
+      }
+
+      public void doWork()
+      {
+         var rows = ExceptionAssert.NotNull(_rows.rows);
+         IBlueRow cand_row = rows[_rows.pointer];
+         string cand_ts_event = cand_row.get<string>("ts_event");
+         DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
+         TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
+         TimeSpan ts_clock = DateTime.Now - _last_fire_time.last_firetime_clock;
+         if (ts_clock >= cand_ts_span_file)
+         {
+            _rows.pointer++;
+            _last_fire_time.last_firetime_clock = DateTime.Now;
+            _last_fire_time.last_fireitme_file = cand_ts_event_dt;
+            BlueTable bt_out = new BlueTable(cand_row.schema);
+            bt_out.addRow(cand_row);
+            var dp_out = BlueDataUtility.build_dp(bt_out, myLastValue as IDataPacket);
+            myFireEvent(dp_out);
+         }
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Data/SQLiteHelper.cs

@@ -0,0 +1,144 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SQLite;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace BlueOrb.Internal
+{
+
+   public static class SQLiteHelper
+   {
+      public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+         ? "C:\\BlueOrb_Data\\"
+         : Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty, "BlueOrb_Data");
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query)
+      {
+         // Create a new DataTable to store the results
+         DataTable dataTable = new DataTable();
+
+         // Create and open a SQLite connection using the provided connection string
+         using (SQLiteConnection connection = new SQLiteConnection(connection_string))
+         {
+            try
+            {
+               connection.Open();
+
+               // Create a SQLite command with the provided query
+               using (SQLiteCommand command = new SQLiteCommand(query, connection))
+               {
+                  // Use SQLiteDataAdapter to fill the DataTable
+                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command))
+                  {
+                     adapter.Fill(dataTable);
+                  }
+               }
+            }
+            catch (Exception ex)
+            {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         // Return the populated DataTable
+         return dataTable;
+      }
+
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query)
+      {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString))
+         {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader())
+            {
+               while (reader.Read())
+               {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++)
+                  {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
+      }
+
+      public static List<Dictionary<string, object>> get_table(string connectionString, string query)
+      {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString))
+         {
+            try
+            {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader())
+               {
+                  while (reader.Read())
+                  {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++)
+                     {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            }
+            catch (Exception ex)
+            {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
+      }
+
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query)
+      {
+         using (var connection = new SQLiteConnection(connectionString))
+         {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader())
+            {
+               while (reader.Read())
+               {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++)
+                  {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
+      }
+   }
+}
+

src/DataStreamSuite/BlueOrbOperatorStreams/Internal/DSLStream.cs

@@ -0,0 +1,43 @@
+namespace BlueOrb.Internal
+{
+   using bts;
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+   using System.Xml;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream
+   {
+      private readonly string _dsl;
+      public DSLStream(string dsl)
+      {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         // Remove all colons
+         string sanitizedDsl = dsl.Replace(":", "COLON");
+         _dsl = dsl;
+         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(string stream_expression)
+      {
+         try
+         {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         }
+         catch (Exception ex)
+         {
+            //send to blueorb
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Internal/MillisecondStream.cs

@@ -0,0 +1,103 @@
+using System;
+
+using BCG.Utilities.General;
+using BCG;
+using bts;
+
+namespace BlueOrb.Internal
+{
+   [DataStreamUserRespresentation("MillisecondStream")]
+   public class MillisecondStream : ViewStream, IWorker
+   {
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private readonly TimeSpan _period;
+      private readonly int _size;
+      private readonly ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema;
+      private bts.CircularBufferTable _table;
+
+      public MillisecondStream(int interval_msecs, int size)
+      : this(ViewStreamDatabase.instance, interval_msecs, size)
+      {
+      }
+
+      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size)
+      {
+         _vsd = vsd;
+         _period = TimeSpan.FromMilliseconds(interval_msecs);
+         _size = size;
+         _schema = BlueDataUtility.form_schema("Milliseconds",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("time", TypeCode.DateTime),
+               Tuple.Create("display_time", TypeCode.String));
+         _table = new CircularBufferTable(_schema, _size);
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         _vsd.removeWorker(this);
+      }
+
+      private int _id = 0;
+      public void doWork()
+      {
+         if (DateTime.Now - _last_fire_time >= _period)
+         {
+            _last_fire_time = DateTime.Now;
+            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
+            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
+            _table.add_row(nr);
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+   }
+   internal class MSecondStream : TimeUnitStream
+   {
+      public MSecondStream(int intervalInms, int size)
+         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
+
+      protected override DateTime myRoundTime(DateTime dt) { return dt; }
+   }
+
+   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper
+   {
+      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
+                                                               new ColumnInfo("id", 0, typeof(int)),
+                                                               new ColumnInfo("time", 1, typeof(string)));
+      public DisplayableMillisecondStream(int intervalInMs, int sz)
+      {
+         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
+      }
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         DateTime time = row.get_datetime("time");
+         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+         return new BlueRow(mySchema, row["id"], time_txt);
+      }
+
+      #endregion
+
+      private string myToString(DateTime dt)
+      {
+         string sep = ":";
+         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Signals/BarBuilderStream.cs

@@ -1,4 +1,5 @@
-namespace BlueOrb.Operators.Signal {
+namespace BlueOrb.Operators.Signal
+{
    using System;
    using System.Collections.Concurrent;
    using System.IO.MemoryMappedFiles;
@@ -6,16 +7,18 @@ namespace BlueOrb.Operators.Signal {
    using System.Text.Json.Serialization;
    using System.Text.Json;
    using System.Data;
-   using bts;
 
-   public enum BarType {
+
+   public enum BarType
+   {
       TIME,
       TICK,
       VOLUME,
       DOLLAR,
    }
 
-   public class BarBuilderSchemaInfo {
+   public class BarBuilderSchemaInfo
+   {
       [JsonPropertyName("price_col")]
       public string price_col { get; set; } = "trade_price";
 
@@ -27,12 +30,13 @@ namespace BlueOrb.Operators.Signal {
    }
 
 
-   public class SingleBar : IBlueRow {
+   public class SingleBar : IBlueRow
+   {
       private BarBuilderStream _parent_str;
       private IBlueTableSchema _schema;
-      // this is the live bar thats being updated
       private MutableBlueRow _row;
-      public SingleBar(BarBuilderStream parent_str) {
+      public SingleBar(BarBuilderStream parent_str)
+      {
          _parent_str = parent_str;
          _schema = BlueDataUtility.form_schema(
             "bar_builder_single_bar",
@@ -51,7 +55,7 @@ namespace BlueOrb.Operators.Signal {
             Tuple.Create("vwap", TypeCode.Double),
             Tuple.Create("is_active", TypeCode.Boolean)
          );
-         reset_new_bar();
+         _row = new MutableBlueRow(_schema);
       }
 
       public IBlueTableSchema schema => _row.schema;
@@ -61,54 +65,40 @@ namespace BlueOrb.Operators.Signal {
       public object this[string col_name] => _row[col_name];
 
 
-      void reset_new_bar() {
-         _row = new MutableBlueRow(_schema);
-         _row.setValue("id", _parent_str.get_new_id());
-         _row.setValue("volume", 0.0);
-         _row.setValue("notional", 0.0);
-         _row.setValue("trade_count", 0);
-         _row.setValue("n_events", 0);
-         _row.setValue("is_active", true);
-      }
-      bool is_virgin_bar() => _row.get<int>("n_events", 0) == 0;
-      
-      public IBlueRow add_event(IBlueRow r) {
-         IBlueRow? close_current_bar_if_needed(IBlueRow input_row) {
-            if (_parent_str._bar_type != BarType.TIME || is_virgin_bar()) {
-               return null;
-            }
-
-            var ts = input_row.get_datetime(_parent_str._schema_info.timestamp_col, DateTime.UtcNow).ToUniversalTime();
-            var bar_end_ts = _row.get_datetime("bar_end_ts", DateTime.MaxValue);
-            if (ts < bar_end_ts) {
-               return null;
-            }
-
-            _row.setValue("is_active", false);
-            IBlueRow closed_row = new BlueRow(_row);
-            reset_new_bar();
-            return closed_row;
+      public IBlueRow add_event(IBlueRow r)
+      {
+         // initiaizes _row
+         void reset_new_bar()
+         {
+            _row = new MutableBlueRow(_schema);
+            _row.setValue("id", _parent_str.get_new_id());
+            _row.setValue("volume", 0.0);
+            _row.setValue("notional", 0.0);
+            _row.setValue("trade_count", 0);
+            _row.setValue("n_events", 0);
+            _row.setValue("is_active", true);
          }
 
-         void absorb_into_row(IBlueRow input_row) {
+         void absorb_into_row(IBlueRow input_row)
+         {
             var ts = input_row.get_datetime(_parent_str._schema_info.timestamp_col, DateTime.UtcNow).ToUniversalTime();
-            var price = input_row.get<double>(_parent_str._schema_info.price_col, Double.NaN);
-            var volume = input_row.get<double>(_parent_str._schema_info.volume_col, Double.NaN);
+            var price = input_row.get<double>(_parent_str._schema_info.price_col, 0.0);
+            var volume = input_row.get<double>(_parent_str._schema_info.volume_col, 0.0);
             var notional = price * volume;
 
-            if (is_virgin_bar()) {
-               // mean absorb_into_row has never been called into the _row. It rests in its 
-               // just opened state
-               // this row was just opened and has not absorbed an input row
-               Log.assert(_row.is_null("bar_start_ts"));
+            if (_row.is_null("bar_start_ts"))
+            {
                DateTime bar_start_ts;
                DateTime bar_end_ts;
-               if (_parent_str._bar_type == BarType.TIME) {
+               if (_parent_str._bar_type == BarType.TIME)
+               {
                   var bucket_ticks = (long)(Math.Max(1.0, _parent_str._bar_size) * TimeSpan.TicksPerSecond);
                   var start_ticks = (ts.Ticks / bucket_ticks) * bucket_ticks;
                   bar_start_ts = new DateTime(start_ticks, DateTimeKind.Utc);
                   bar_end_ts = bar_start_ts.AddTicks(bucket_ticks);
-               } else {
+               }
+               else
+               {
                   bar_start_ts = ts;
                   bar_end_ts = ts;
                }
@@ -117,7 +107,9 @@ namespace BlueOrb.Operators.Signal {
                _row.setValue("open", price);
                _row.setValue("high", price);
                _row.setValue("low", price);
-            } else {
+            }
+            else
+            {
                var high = _row.get<double>("high", price);
                var low = _row.get<double>("low", price);
                _row.setValue("high", Math.Max(high, price));
@@ -135,49 +127,66 @@ namespace BlueOrb.Operators.Signal {
             _row.setValue("vwap", running_volume > 0.0 ? running_notional / running_volume : price);
          }
 
-         bool is_closed() {
+         bool is_closed()
+         {
             bool closed;
-            if (_parent_str._bar_type == BarType.TIME) {
+            if (_parent_str._bar_type == BarType.TIME)
+            {
                var ts = _row.get_datetime("ts_event", DateTime.MinValue);
                var bar_end_ts = _row.get_datetime("bar_end_ts", DateTime.MaxValue);
                closed = ts >= bar_end_ts;
-            } else if (_parent_str._bar_type == BarType.TICK) {
+            }
+            else if (_parent_str._bar_type == BarType.TICK)
+            {
                closed = _row.get<int>("n_events", 0) >= _parent_str._bar_size;
-            } else if (_parent_str._bar_type == BarType.VOLUME) {
+            }
+            else if (_parent_str._bar_type == BarType.VOLUME)
+            {
                closed = _row.get<double>("volume", 0.0) >= _parent_str._bar_size;
-            } else {
+            }
+            else
+            {
                closed = _row.get<double>("notional", 0.0) >= _parent_str._bar_size;
             }
             _row.setValue("is_active", !closed);
             return closed;
          }
 
-         Log.assert(_row != null);
-         var closed_row = close_current_bar_if_needed(r);
-         absorb_into_row(r);
-         if (is_closed()) {
-            closed_row = new BlueRow(_row);
+         if (_row == null)
+         {
             reset_new_bar();
          }
-         return closed_row;
+         absorb_into_row(r);
+         if (is_closed())
+         {
+            IBlueRow closed_row = new BlueRow(_row);
+            reset_new_bar();
+            return closed_row;
+         }
+         return null;
       }
 
    }
 
 
-   public class BarState : IBlueRowCollection {
+   public class BarState : IBlueRowCollection
+   {
       private SingleBar _curr_row;
       private BlueTable _state;
 
-      public BarState(BarBuilderStream parent_str) {
+      public BarState(BarBuilderStream parent_str)
+      {
          _curr_row = new SingleBar(parent_str);
          _state = new BlueTable(_curr_row.schema);
       }
 
-      public bool add_event(IBlueRow r) {
+      public bool add_event(IBlueRow r)
+      {
          var closed_bar = _curr_row.add_event(r);
-         if (closed_bar is not null) {
-            // new bar opened
+         if (closed_bar is not null)
+         {
+            // curr bar is closed, only then its added
+            // when bars are evolving we dont add it
             _state.addRow(closed_bar);
             return true;
          }
@@ -196,7 +205,8 @@ namespace BlueOrb.Operators.Signal {
    }
 
    [DataStreamUserRespresentation("BarBuilderStream")]
-   public class BarBuilderStream : ViewStream {
+   public class BarBuilderStream : ViewStream
+   {
       readonly ViewStream _price_stream;
       internal readonly BarType _bar_type;
       internal readonly double _bar_size;
@@ -208,7 +218,8 @@ namespace BlueOrb.Operators.Signal {
                                BarType bar_type,
                                double bar_size,
                                bool show_open_bars,
-                               BarBuilderSchemaInfo schema_info) {
+                               BarBuilderSchemaInfo schema_info)
+      {
          _price_stream = price_stream;
          _bar_type = bar_type;
          _bar_size = bar_size;
@@ -218,7 +229,8 @@ namespace BlueOrb.Operators.Signal {
       }
 
       int _id = 0;
-      internal int get_new_id() {
+      internal int get_new_id()
+      {
          return _id++;
       }
 
@@ -228,10 +240,11 @@ namespace BlueOrb.Operators.Signal {
                                string schema_info_json)
                                :
                                this(price_stream,
-                                     _parse_bar_type(bar_type),
+                                     Enum.Parse<BarType>(bar_type),
                                      bar_size,
                                      true,
-                                    _parse_and_validate_schema_info_json(schema_info_json)) {
+                                    _parse_and_validate_schema_info_json(schema_info_json))
+      {
       }
 
       public BarBuilderStream(ViewStream price_stream,
@@ -242,29 +255,33 @@ namespace BlueOrb.Operators.Signal {
                                string timestamp_col)
                                :
                                this(price_stream,
-                                     _parse_bar_type(bar_type),
+                                     Enum.Parse<BarType>(bar_type),
                                      bar_size,
                                      true,
-                                     _create_and_validate_schema_info(price_col, volume_col, timestamp_col)) {
+                                     _create_and_validate_schema_info(price_col, volume_col, timestamp_col))
+      {
       }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _price_stream.onChange += _on_price_stream_change;
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _price_stream.onChange -= _on_price_stream_change;
       }
 
-      void _on_price_stream_change(object dp_obj) {
+      void _on_price_stream_change(object dp_obj)
+      {
          DataPacket dp = dp_obj as DataPacket;
          var sz_0 = _state.Count;
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
             _state.add_event(r);
          }
-         if (_state.Count == sz_0)
-            // need to check if we fire open bar changes, or only on bar clsoe
-            return;
+         if (_state.Count != sz_0)
+         // we will check for _show_open_bars here 
          {
             var dp_out = BlueDataUtility.build_dp_inc(_state, myLastValue as DataPacket, false);
             myFireEvent(dp_out);
@@ -272,41 +289,10 @@ namespace BlueOrb.Operators.Signal {
       }
 
 
-      static BarType _parse_bar_type(string bar_type) {
-         if (string.IsNullOrWhiteSpace(bar_type)) {
-            throw new ArgumentException("bar_type cannot be null or empty.", nameof(bar_type));
-         }
-
-         var normalized = bar_type.Trim();
-         if (Enum.TryParse<BarType>(normalized, ignoreCase: true, out var exact_match)) {
-            return exact_match;
-         }
-
-         BarType? substring_match = null;
-         foreach (var candidate in Enum.GetValues<BarType>()) {
-            var candidate_name = candidate.ToString();
-            bool is_match =
-               candidate_name.Contains(normalized, StringComparison.OrdinalIgnoreCase) ||
-               normalized.Contains(candidate_name, StringComparison.OrdinalIgnoreCase);
-            if (!is_match) {
-               continue;
-            }
-
-            if (substring_match is not null) {
-               throw new ArgumentException($"bar_type '{bar_type}' is ambiguous.", nameof(bar_type));
-            }
-            substring_match = candidate;
-         }
-
-         if (substring_match is not null) {
-            return substring_match.Value;
-         }
-
-         throw new ArgumentException($"Unknown bar_type '{bar_type}'.", nameof(bar_type));
-      }
-
-      static BarBuilderSchemaInfo _parse_and_validate_schema_info_json(string schema_info_json) {
-         if (string.IsNullOrWhiteSpace(schema_info_json)) {
+      static BarBuilderSchemaInfo _parse_and_validate_schema_info_json(string schema_info_json)
+      {
+         if (string.IsNullOrWhiteSpace(schema_info_json))
+         {
             throw new ArgumentException("schema_info_json cannot be null or empty.", nameof(schema_info_json));
          }
 
@@ -318,8 +304,10 @@ namespace BlueOrb.Operators.Signal {
       static BarBuilderSchemaInfo _create_and_validate_schema_info(
          string price_col,
          string volume_col,
-         string timestamp_col) {
-         var schema_info = new BarBuilderSchemaInfo {
+         string timestamp_col)
+      {
+         var schema_info = new BarBuilderSchemaInfo
+         {
             price_col = price_col,
             volume_col = volume_col,
             timestamp_col = timestamp_col,
@@ -328,24 +316,30 @@ namespace BlueOrb.Operators.Signal {
          return schema_info;
       }
 
-      static void _validate_schema_info(BarBuilderSchemaInfo? schema_info) {
-         if (schema_info is null) {
+      static void _validate_schema_info(BarBuilderSchemaInfo? schema_info)
+      {
+         if (schema_info is null)
+         {
             throw new ArgumentException("schema_info_json did not deserialize into BarBuilderSchemaInfo.");
          }
 
-         if (string.IsNullOrWhiteSpace(schema_info.price_col)) {
+         if (string.IsNullOrWhiteSpace(schema_info.price_col))
+         {
             throw new ArgumentException("BarBuilderSchemaInfo.PriceCol cannot be null or empty.");
          }
-         if (string.IsNullOrWhiteSpace(schema_info.volume_col)) {
+         if (string.IsNullOrWhiteSpace(schema_info.volume_col))
+         {
             throw new ArgumentException("BarBuilderSchemaInfo.VolumeCol cannot be null or empty.");
          }
-         if (string.IsNullOrWhiteSpace(schema_info.timestamp_col)) {
+         if (string.IsNullOrWhiteSpace(schema_info.timestamp_col))
+         {
             throw new ArgumentException("BarBuilderSchemaInfo.TimestampCol cannot be null or empty.");
          }
 
          if (schema_info.price_col.Equals(schema_info.volume_col, StringComparison.OrdinalIgnoreCase) ||
              schema_info.price_col.Equals(schema_info.timestamp_col, StringComparison.OrdinalIgnoreCase) ||
-             schema_info.volume_col.Equals(schema_info.timestamp_col, StringComparison.OrdinalIgnoreCase)) {
+             schema_info.volume_col.Equals(schema_info.timestamp_col, StringComparison.OrdinalIgnoreCase))
+         {
             throw new ArgumentException("BarBuilderSchemaInfo columns must be distinct.");
          }
       }

src/DataStreamSuite/DSPublisher/Configurations/.gitkeep

@@ -1 +0,0 @@
-

src/DataStreamSuite/DSPublisher/DSLWSClient/dsl_websocket_client.py

@@ -1,168 +0,0 @@
-"""
-Simple DSPublisher DSL websocket client.
-
-Example:
-   python dsl_websocket_client.py --dsl "HeikinAshiStream(...)"
-   python dsl_websocket_client.py --dsl-file query.dsl
-"""
-
-import argparse
-import asyncio
-import json
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-
-DEFAULT_DSL = 'RSIUIStream("ES", "TIME", 5, 7)'
-DEFAULT_URL = "ws://localhost:19093/ws/"
-
-
-def load_dsl(dsl: str | None, dsl_file: str | None) -> str:
-   if dsl and dsl_file:
-      raise ValueError("Pass either --dsl or --dsl-file, not both.")
-
-   if dsl_file:
-      return Path(dsl_file).read_text(encoding="utf-8").strip()
-
-   if dsl:
-      return dsl.strip()
-
-   if DEFAULT_DSL.startswith("#"):
-      raise ValueError("No DSL provided. Fill in DEFAULT_DSL, or pass --dsl / --dsl-file.")
-
-   return DEFAULT_DSL
-
-
-def load_config_path(config_path: str | None) -> Path | None:
-   if config_path:
-      return Path(config_path)
-
-   environment = os.environ.get("DOTNET_ENVIRONMENT")
-   config_file_name = "appsettings.json" if not environment else f"appsettings.{environment}.json"
-   script_dir = Path(__file__).resolve().parent
-   candidate_dirs = [
-      script_dir,
-      script_dir.parent,
-      Path.cwd(),
-   ]
-
-   for candidate_dir in candidate_dirs:
-      candidate_path = candidate_dir / config_file_name
-      if candidate_path.exists():
-         return candidate_path
-
-   return None
-
-
-def load_url(url: str | None, config_path: str | None) -> str:
-   if url:
-      return url
-
-   resolved_config_path = load_config_path(config_path)
-   if not resolved_config_path:
-      print(f"Config file not found; using default url: {DEFAULT_URL}")
-      return DEFAULT_URL
-
-   config = json.loads(resolved_config_path.read_text(encoding="utf-8"))
-   dsl_api_server_config = config.get("DSLAPIServer", {})
-   ip = dsl_api_server_config.get("Ip", "localhost")
-   port = int(dsl_api_server_config.get("Port", 19093))
-   ws_url = f"ws://{ip}:{port}/ws/"
-   print(f"Loaded DSPublisher DSL websocket url from {resolved_config_path}: {ws_url}")
-   return ws_url
-
-
-async def send_subscription(websocket, dsl: str) -> None:
-   payload = {
-      "type": "subscribe",
-      "dsl": dsl,
-   }
-   await websocket.send(json.dumps(payload))
-   print("Sent subscribe request:")
-   print(json.dumps(payload, indent=2))
-
-
-async def send_unsubscribe(websocket, subscription_id: str) -> None:
-   payload = {
-      "type": "unsubscribe",
-      "subscriptionId": subscription_id,
-   }
-   await websocket.send(json.dumps(payload))
-   print("Sent unsubscribe request:")
-   print(json.dumps(payload, indent=2))
-
-
-def print_server_message(raw_message: str) -> str | None:
-   try:
-      message = json.loads(raw_message)
-   except json.JSONDecodeError:
-      print(raw_message)
-      return None
-
-   print("Received server message:")
-   print(json.dumps(message, indent=2))
-   return message.get("subscriptionId")
-
-
-async def run_client(url: str, dsl: str) -> None:
-   subscription_id: str | None = None
-
-   async with websockets.connect(url) as websocket:
-      print(f"Connected to DSPublisher DSL websocket: {url}")
-      await send_subscription(websocket, dsl)
-
-      try:
-         async for raw_message in websocket:
-            latest_subscription_id = print_server_message(raw_message)
-            subscription_id = subscription_id or latest_subscription_id
-      finally:
-         if subscription_id:
-            await send_unsubscribe(websocket, subscription_id)
-
-
-def main() -> int:
-   parser = argparse.ArgumentParser(
-      description="Subscribe to a DSPublisher DSL websocket query and print responses."
-   )
-   parser.add_argument(
-      "--url",
-      help="DSPublisher DSL websocket URL. Overrides appsettings DSLAPIServer config.",
-   )
-   parser.add_argument(
-      "--config",
-      help="Path to DSPublisher appsettings JSON. Defaults to appsettings[.$DOTNET_ENVIRONMENT].json near this script.",
-   )
-   parser.add_argument(
-      "--dsl",
-      help="DSL expression to subscribe to.",
-   )
-   parser.add_argument(
-      "--dsl-file",
-      help="Path to a text file containing the DSL expression.",
-   )
-   args = parser.parse_args()
-
-   try:
-      dsl = load_dsl(args.dsl, args.dsl_file)
-      url = load_url(args.url, args.config)
-      asyncio.run(run_client(url, dsl))
-      return 0
-   except KeyboardInterrupt:
-      print("stopped")
-      return 0
-   except Exception as ex:
-      print(f"error: {ex}")
-      return 1
-
-
-if __name__ == "__main__":
-   raise SystemExit(main())

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLAPIWSListener.cs

@@ -1,264 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Net;
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Threading;
-using System.Threading.Tasks;
-
-using BCG;
-using Bts.DataStreamCore;
-
-/// <summary>
-/// Websocket-facing API boundary for DSL clients. It opens the listener port,
-/// accepts subscribe/unsubscribe messages, and sends DataPacket JSON back to clients.
-/// Instantiated by DSPAPIServer.
-/// </summary>
-public sealed class DSLAPIWSListener : IDisposable {
-   readonly HttpListener listener = new();
-   readonly IDSLBookkeeper bookkeeper;
-   readonly string prefix;
-   readonly ConcurrentDictionary<string, ClientConnection> clients_by_id = new();
-   CancellationTokenSource? cancellation_token_source;
-   Task? listener_task;
-   bool disposed;
-
-   sealed class ClientConnection {
-      public required string client_id { get; init; }
-      public required WebSocket web_socket { get; init; }
-      public HashSet<Guid> subscription_ids { get; } = new();
-      public SemaphoreSlim send_lock { get; } = new(1, 1);
-      public DataPacketJsonSerializer serializer { get; } = new();
-   }
-
-   public DSLAPIWSListener(string prefix, IDSLBookkeeper bookkeeper) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(prefix);
-      ArgumentNullException.ThrowIfNull(bookkeeper);
-      this.prefix = prefix;
-      this.bookkeeper = bookkeeper;
-   }
-
-   public Task StartAsync(CancellationToken cancellation_token = default) {
-      _throw_if_disposed();
-      if (listener_task is not null) {
-         return Task.CompletedTask;
-      }
-
-      cancellation_token_source = CancellationTokenSource.CreateLinkedTokenSource(cancellation_token);
-      listener.Prefixes.Add(prefix);
-      listener.Start();
-      listener_task = Task.Run(() => _listen_loop(cancellation_token_source.Token), CancellationToken.None);
-      return Task.CompletedTask;
-   }
-
-   public async Task StopAsync() {
-      if (cancellation_token_source is null) {
-         return;
-      }
-
-      cancellation_token_source.Cancel();
-      if (listener.IsListening) {
-         listener.Stop();
-      }
-
-      foreach (var client_connection in clients_by_id.Values) {
-         await _cleanup_client(client_connection);
-      }
-
-      if (listener_task is not null) {
-         await listener_task;
-         listener_task = null;
-      }
-
-      cancellation_token_source.Dispose();
-      cancellation_token_source = null;
-   }
-
-   public void Dispose() {
-      if (disposed) {
-         return;
-      }
-
-      StopAsync().GetAwaiter().GetResult();
-      listener.Close();
-      disposed = true;
-   }
-
-   async Task _listen_loop(CancellationToken cancellation_token) {
-      while (!cancellation_token.IsCancellationRequested) {
-         HttpListenerContext context;
-         try {
-            context = await listener.GetContextAsync();
-         } catch (HttpListenerException) when (cancellation_token.IsCancellationRequested || !listener.IsListening) {
-            break;
-         } catch (ObjectDisposedException) {
-            break;
-         }
-
-         _ = Task.Run(() => _handle_request(context, cancellation_token), cancellation_token);
-      }
-   }
-
-   async Task _handle_request(HttpListenerContext context, CancellationToken cancellation_token) {
-      if (!context.Request.IsWebSocketRequest) {
-         context.Response.StatusCode = (int)HttpStatusCode.BadRequest;
-         context.Response.Close();
-         return;
-      }
-
-      HttpListenerWebSocketContext web_socket_context = await context.AcceptWebSocketAsync(null);
-      var client_connection = new ClientConnection {
-         client_id = Guid.NewGuid().ToString("N"),
-         web_socket = web_socket_context.WebSocket,
-      };
-
-      clients_by_id[client_connection.client_id] = client_connection;
-      try {
-         await _read_messages(client_connection, cancellation_token);
-      } finally {
-         await _cleanup_client(client_connection);
-      }
-   }
-
-   async Task _read_messages(ClientConnection client_connection, CancellationToken cancellation_token) {
-      var buffer = new byte[64 * 1024];
-      while (client_connection.web_socket.State == WebSocketState.Open && !cancellation_token.IsCancellationRequested) {
-         var result = await client_connection.web_socket.ReceiveAsync(buffer, cancellation_token);
-         if (result.MessageType == WebSocketMessageType.Close) {
-            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "ok", cancellation_token);
-            break;
-         }
-
-         var json = Encoding.UTF8.GetString(buffer, 0, result.Count);
-         await _handle_message(client_connection, json, cancellation_token);
-      }
-   }
-
-   async Task _handle_message(ClientConnection client_connection, string json, CancellationToken cancellation_token) {
-      DSLWebSocketRequest? request;
-      try {
-         request = JsonSerializer.Deserialize<DSLWebSocketRequest>(json);
-      } catch (JsonException ex) {
-         await _send_error(client_connection, $"Invalid request JSON: {ex.Message}", cancellation_token);
-         return;
-      }
-
-      if (request?.type is null) {
-         await _send_error(client_connection, "Request type is required.", cancellation_token);
-         return;
-      }
-
-      if (string.Equals(request.type, "subscribe", StringComparison.OrdinalIgnoreCase)) {
-         await _handle_subscribe(client_connection, request, cancellation_token);
-         return;
-      }
-
-      if (string.Equals(request.type, "unsubscribe", StringComparison.OrdinalIgnoreCase)) {
-         await _handle_unsubscribe(client_connection, request, cancellation_token);
-         return;
-      }
-
-      await _send_error(client_connection, $"Unknown request type: {request.type}", cancellation_token);
-   }
-
-   async Task _handle_subscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
-      if (string.IsNullOrWhiteSpace(request.dsl)) {
-         await _send_error(client_connection, "Subscribe request requires dsl.", cancellation_token);
-         return;
-      }
-
-      Guid subscription_id = Guid.Empty;
-      subscription_id = bookkeeper.Subscribe(request.dsl, packet => {
-         if (subscription_id == Guid.Empty) {
-            return;
-         }
-         _ = _send_packet(client_connection, subscription_id, packet, CancellationToken.None);
-      }, client_connection.client_id);
-
-      client_connection.subscription_ids.Add(subscription_id);
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "subscribed",
-         subscription_id = subscription_id,
-      }, cancellation_token);
-   }
-
-   async Task _handle_unsubscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
-      if (request.subscription_id is null) {
-         await _send_error(client_connection, "Unsubscribe request requires subscriptionId.", cancellation_token);
-         return;
-      }
-
-      var subscription_id = request.subscription_id.Value;
-      if (!client_connection.subscription_ids.Remove(subscription_id)) {
-         await _send_error(client_connection, $"Subscription does not belong to this client: {subscription_id}", cancellation_token);
-         return;
-      }
-
-      bookkeeper.Unsubscribe(subscription_id);
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "unsubscribed",
-         subscription_id = subscription_id,
-      }, cancellation_token);
-   }
-
-   async Task _send_packet(ClientConnection client_connection, Guid subscription_id, DataPacket packet, CancellationToken cancellation_token) {
-      var packet_json = client_connection.serializer.Serialize(packet);
-      using var packet_document = JsonDocument.Parse(packet_json ?? "{}");
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "data",
-         subscription_id = subscription_id,
-         packet = packet_document.RootElement.Clone(),
-      }, cancellation_token);
-   }
-
-   Task _send_error(ClientConnection client_connection, string message, CancellationToken cancellation_token) =>
-      _send_response(client_connection, new DSLWebSocketResponse {
-         type = "error",
-         message = message,
-      }, cancellation_token);
-
-   async Task _send_response(ClientConnection client_connection, DSLWebSocketResponse response, CancellationToken cancellation_token) {
-      if (client_connection.web_socket.State != WebSocketState.Open) {
-         bookkeeper.DisposeClient(client_connection.client_id);
-         return;
-      }
-
-      var json = JsonSerializer.Serialize(response);
-      var bytes = Encoding.UTF8.GetBytes(json);
-      await client_connection.send_lock.WaitAsync(cancellation_token);
-      try {
-         await client_connection.web_socket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellation_token);
-      } catch {
-         bookkeeper.DisposeClient(client_connection.client_id);
-         throw;
-      } finally {
-         client_connection.send_lock.Release();
-      }
-   }
-
-   async Task _cleanup_client(ClientConnection client_connection) {
-      if (clients_by_id.TryRemove(client_connection.client_id, out _)) {
-         bookkeeper.DisposeClient(client_connection.client_id);
-      }
-
-      if (client_connection.web_socket.State == WebSocketState.Open || client_connection.web_socket.State == WebSocketState.CloseReceived) {
-         try {
-            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "cleanup", CancellationToken.None);
-         } catch {
-         }
-      }
-
-      client_connection.send_lock.Dispose();
-      client_connection.web_socket.Dispose();
-   }
-
-   void _throw_if_disposed() {
-      if (disposed) {
-         throw new ObjectDisposedException(nameof(DSLAPIWSListener));
-      }
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLBookkeeper.cs

@@ -1,216 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Collections.Generic;
-
-using BCG;
-using Bts.DataStreamCore;
-using bts.BlueOrb;
-using bts;
-
-/// <summary>
-/// Tracks active DSL subscriptions and shared DSLStream instances. Multiple clients
-/// subscribing to the same DSL reuse one stream, and DoWork dispatches queued packets.
-/// Instantiated by DSPAPIServer.
-/// </summary>
-public sealed class DSLBookkeeper : IDSLBookkeeper {
-   sealed class QueryState {
-      public required string dsl { get; init; }
-      public required DSLStream stream { get; init; }
-      public DataStream.ChangeDelegate? stream_handler { get; set; }
-      public HashSet<Guid> subscription_ids { get; } = new();
-      public Queue<DataPacket> pending_packets { get; } = new();
-   }
-
-   sealed class SubscriptionState {
-      public required Guid subscription_id { get; init; }
-      public required string client_id { get; init; }
-      public required string dsl_key { get; init; }
-      public required Action<DataPacket> on_data { get; init; }
-   }
-
-   readonly object lock_obj = new();
-   readonly Dictionary<string, QueryState> queries_by_dsl = new(StringComparer.Ordinal);
-   readonly Dictionary<Guid, SubscriptionState> subscriptions_by_id = new();
-   readonly Dictionary<string, HashSet<Guid>> subscriptions_by_client = new(StringComparer.Ordinal);
-   bool disposed;
-
-   public Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(dsl);
-      ArgumentNullException.ThrowIfNull(on_data);
-      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
-
-      lock (lock_obj) {
-         _throw_if_disposed();
-
-         var dsl_key = _normalize_dsl(dsl);
-         if (!queries_by_dsl.TryGetValue(dsl_key, out var query_state)) {
-            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} creating new DSLStream dsl: {dsl_key}");
-            query_state = _create_query_state(dsl_key);
-            queries_by_dsl.Add(dsl_key, query_state);
-         } else {
-            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} reusing existing DSLStream subscription_count: {query_state.subscription_ids.Count} dsl: {dsl_key}");
-         }
-
-         var subscription_id = Guid.NewGuid();
-         var subscription_state = new SubscriptionState {
-            subscription_id = subscription_id,
-            client_id = client_id,
-            dsl_key = dsl_key,
-            on_data = on_data,
-         };
-
-         subscriptions_by_id.Add(subscription_id, subscription_state);
-         query_state.subscription_ids.Add(subscription_id);
-
-         if (!subscriptions_by_client.TryGetValue(client_id, out var client_subscriptions)) {
-            client_subscriptions = new();
-            subscriptions_by_client.Add(client_id, client_subscriptions);
-         }
-         client_subscriptions.Add(subscription_id);
-
-         _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} subscription_id: {subscription_id} active_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {dsl_key}");
-         return subscription_id;
-      }
-   }
-
-   public void Unsubscribe(Guid subscription_id) {
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-         _unsubscribe_locked(subscription_id);
-      }
-   }
-
-   public void DisposeClient(string client_id) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
-
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         if (!subscriptions_by_client.TryGetValue(client_id, out var subscription_ids)) {
-            return;
-         }
-
-         foreach (var subscription_id in subscription_ids.ToArray()) {
-            _unsubscribe_locked(subscription_id);
-         }
-      }
-   }
-
-   public void DoWork() {
-      List<(Action<DataPacket> on_data, DataPacket packet)> callbacks = new();
-
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         foreach (var query_state in queries_by_dsl.Values) {
-            while (query_state.pending_packets.TryDequeue(out var packet)) {
-               foreach (var subscription_id in query_state.subscription_ids) {
-                  if (subscriptions_by_id.TryGetValue(subscription_id, out var subscription_state)) {
-                     callbacks.Add((subscription_state.on_data, packet));
-                  }
-               }
-            }
-         }
-      }
-
-      foreach (var callback in callbacks) {
-         callback.on_data(callback.packet);
-      }
-   }
-
-   public void Dispose() {
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         foreach (var query_state in queries_by_dsl.Values) {
-            _dispose_query_state(query_state);
-         }
-
-         queries_by_dsl.Clear();
-         subscriptions_by_id.Clear();
-         subscriptions_by_client.Clear();
-         disposed = true;
-      }
-   }
-
-   QueryState _create_query_state(string dsl_key) {
-      _log_lifecycle($"DSLBookkeeper._create_query_state creating DSLStream dsl: {dsl_key}");
-      var stream = new DSLStream(dsl_key);
-      var query_state = new QueryState {
-         dsl = dsl_key,
-         stream = stream,
-      };
-
-      DataStream.ChangeDelegate stream_handler = new_value => {
-         if (new_value is not DataPacket packet) {
-            return;
-         }
-
-         lock (lock_obj) {
-            if (!disposed) {
-               query_state.pending_packets.Enqueue(packet);
-            }
-         }
-      };
-
-      query_state.stream_handler = stream_handler;
-      stream.onChange += stream_handler;
-      return query_state;
-   }
-
-   void _unsubscribe_locked(Guid subscription_id) {
-      if (!subscriptions_by_id.Remove(subscription_id, out var subscription_state)) {
-         return;
-      }
-
-      if (subscriptions_by_client.TryGetValue(subscription_state.client_id, out var client_subscriptions)) {
-         client_subscriptions.Remove(subscription_id);
-         if (client_subscriptions.Count == 0) {
-            subscriptions_by_client.Remove(subscription_state.client_id);
-         }
-      }
-
-      if (queries_by_dsl.TryGetValue(subscription_state.dsl_key, out var query_state)) {
-         query_state.subscription_ids.Remove(subscription_id);
-         _log_lifecycle($"DSLBookkeeper.Unsubscribe subscription_id: {subscription_id} client_id: {subscription_state.client_id} remaining_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {subscription_state.dsl_key}");
-         if (query_state.subscription_ids.Count == 0) {
-            _log_lifecycle($"DSLBookkeeper.Unsubscribe garbage collecting DSLStream; last subscription removed dsl: {subscription_state.dsl_key}");
-            _dispose_query_state(query_state);
-            queries_by_dsl.Remove(subscription_state.dsl_key);
-         }
-      }
-   }
-
-   static string _normalize_dsl(string dsl) =>
-      dsl.Trim();
-
-   static void _dispose_query_state(QueryState query_state) {
-      _log_lifecycle($"DSLBookkeeper._dispose_query_state disposing DSLStream dsl: {query_state.dsl} pending_packets: {query_state.pending_packets.Count} subscription_count: {query_state.subscription_ids.Count}");
-      if (query_state.stream_handler is not null) {
-         query_state.stream.onChange -= query_state.stream_handler;
-         query_state.stream_handler = null;
-      }
-      query_state.subscription_ids.Clear();
-      query_state.pending_packets.Clear();
-   }
-
-   static void _log_lifecycle(string message) {
-      Console.WriteLine(message);
-      Log.Gen.info(message);
-   }
-
-   void _throw_if_disposed() {
-      if (disposed) {
-         throw new ObjectDisposedException(nameof(DSLBookkeeper));
-      }
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLWebSocketMessages.cs

@@ -1,39 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-/// <summary>
-/// Client-to-server websocket protocol message for subscribing and unsubscribing.
-/// Instantiated by System.Text.Json when DSLAPIWSListener deserializes client messages.
-/// </summary>
-public sealed class DSLWebSocketRequest {
-   [JsonPropertyName("type")]
-   public string? type { get; set; }
-
-   [JsonPropertyName("dsl")]
-   public string? dsl { get; set; }
-
-   [JsonPropertyName("subscriptionId")]
-   public Guid? subscription_id { get; set; }
-}
-
-/// <summary>
-/// Server-to-client websocket protocol message. Data responses carry serialized
-/// DataPacket JSON in the packet field.
-/// Instantiated by DSLAPIWSListener before serializing responses to clients.
-/// </summary>
-public sealed class DSLWebSocketResponse {
-   [JsonPropertyName("type")]
-   public string type { get; set; } = "";
-
-   [JsonPropertyName("subscriptionId")]
-   public Guid? subscription_id { get; set; }
-
-   [JsonPropertyName("message")]
-   public string? message { get; set; }
-
-   [JsonPropertyName("packet")]
-   public JsonElement? packet { get; set; }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSPAPIServer.cs

@@ -1,37 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-
-/// <summary>
-/// Top-level DSPublisher API server. Owns the websocket listener and the DSL
-/// subscription bookkeeper, and exposes the small lifecycle surface Program.cs uses.
-/// Instantiated by Program.cs.
-/// </summary>
-public sealed class DSPAPIServer : IDisposable {
-   readonly DSLBookkeeper bookkeeper;
-   readonly DSLAPIWSListener web_socket_listener;
-
-   public DSPAPIServer(global::DSLAPIServerConfig config) {
-      ArgumentNullException.ThrowIfNull(config);
-      bookkeeper = new DSLBookkeeper();
-      var web_socket_prefix = $"http://{config.Ip}:{config.Port}/ws/";
-      web_socket_listener = new DSLAPIWSListener(web_socket_prefix, bookkeeper);
-      Console.WriteLine($"DSPAPIServer listening for websocket DSL API requests on {web_socket_prefix}");
-   }
-
-   public Task StartAsync(CancellationToken cancellation_token = default) =>
-      web_socket_listener.StartAsync(cancellation_token);
-
-   public Task StopAsync() =>
-      web_socket_listener.StopAsync();
-
-   public void DoWork() =>
-      bookkeeper.DoWork();
-
-   public void Dispose() {
-      web_socket_listener.Dispose();
-      bookkeeper.Dispose();
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/IDSLBookkeeper.cs

@@ -1,18 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-
-using BCG;
-using Bts.DataStreamCore;
-
-/// <summary>
-/// Internal contract between the websocket listener and the DSL subscription owner.
-/// The listener handles transport; the bookkeeper owns stream lifetime and callbacks.
-/// Implemented by DSLBookkeeper and consumed by DSLAPIWSListener.
-/// </summary>
-public interface IDSLBookkeeper : IDisposable {
-   Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id);
-   void Unsubscribe(Guid subscription_id);
-   void DisposeClient(string client_id);
-   void DoWork();
-}

src/DataStreamSuite/DSPublisher/DSPublisher.csproj

@@ -1,31 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\PubSub\PubSub.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ConfigurationProfile Include="appsettings*.json" />
-
-    <None Update="appsettings*.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>DSPublisher.%(Filename)%(Extension)</TargetPath>
-    </None>
-    <None Update="WSPublishers\**\*.py">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="DSLWSClient\**\*.py">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/DSPublisher/DSPublisherConfig.cs

@@ -1,21 +0,0 @@
-using System.Collections.Generic;
-
-public sealed class DSPublisherConfig {
-   public List<StreamPublishingNode> StreamPublishingNodes { get; init; } = new();
-   public List<PythonPublishingNode> PythonPublishingNodes { get; init; } = new();
-}
-
-public sealed class DSLAPIServerConfig {
-   public string Ip { get; init; } = "localhost";
-   public int Port { get; init; } = 19093;
-}
-
-public sealed class StreamPublishingNode {
-   public string Name { get; init; } = string.Empty;
-   public string Expression { get; init; } = string.Empty;
-}
-
-public sealed class PythonPublishingNode {
-   public string PythonScriptLocation { get; init; } = string.Empty;
-   public string Args { get; init; } = string.Empty;
-}

src/DataStreamSuite/DSPublisher/Program.cs

@@ -1,125 +0,0 @@
-using System;
-using System.IO;
-using System.Text.Json;
-using System.Threading;
-
-using BCG;
-using bts;
-using Bts.PubSub;
-using bts.BlueOrb;
-using DSPublisher.DSLWSServer;
-
-Console.WriteLine("Starting DSPublisher");
-
-var configPath = load_config_path();
-Console.WriteLine($"Loading config from: {configPath}");
-if (!File.Exists(configPath)) {
-   Console.WriteLine($"Config file not found: {configPath}");
-   return;
-}
-
-var json = File.ReadAllText(configPath);
-using var doc = JsonDocument.Parse(json);
-
-if (!doc.RootElement.TryGetProperty("DSPublisher", out var dspPublisherElement)) {
-   Console.WriteLine("DSPublisher config section not found");
-   return;
-}
-
-var json_options = new JsonSerializerOptions {
-   PropertyNameCaseInsensitive = true,
-};
-
-var config = dspPublisherElement.Deserialize<DSPublisherConfig>(json_options) ?? new DSPublisherConfig();
-var dsl_api_server_config = load_dsl_api_server_config(doc, json_options);
-
-Console.WriteLine("StreamPublishingNodes:");
-foreach (var node in config.StreamPublishingNodes) {
-   Console.WriteLine($"  Name: {node.Name}");
-   Console.WriteLine($"  Expression: {node.Expression}");
-}
-
-Console.WriteLine("PythonPublishingNodes:");
-foreach (var node in config.PythonPublishingNodes) {
-   Console.WriteLine($"  PythonScriptLocation: {node.PythonScriptLocation}");
-   Console.WriteLine($"  Args: {node.Args}");
-}
-
-var pubsub_config = load_pubsub_config(doc, json_options);
-PubSubConfig.SetGlobalConfig(pubsub_config);
-using var view_stream_database = new ViewStreamDatabase(new SubscriberCoreFactory());
-ViewStreamDatabase.SetGlobalInstance(view_stream_database);
-
-//var pub_table_broker = new PubTableBroker();
-//var stream_publishers = _process_publishing_nodes(config.StreamPublishingNodes, pub_table_broker);
-
-using var dsp_api_server = new DSPAPIServer(dsl_api_server_config);
-await dsp_api_server.StartAsync();
-
-while (true) {
-   view_stream_database.doWork();
-   dsp_api_server.DoWork();
-   Thread.Sleep(TimeSpan.FromMilliseconds(10));
-}
-
-string load_config_path() {
-   var environment = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
-   var baseFileName = string.IsNullOrWhiteSpace(environment)
-      ? "appsettings.json"
-      : $"appsettings.{environment}.json";
-   var dspPublisherFileName = $"DSPublisher.{baseFileName}";
-   var dspPublisherConfigPath = Path.Combine(AppContext.BaseDirectory, dspPublisherFileName);
-   if (File.Exists(dspPublisherConfigPath)) {
-      return dspPublisherConfigPath;
-   }
-
-   return Path.Combine(AppContext.BaseDirectory, baseFileName);
-}
-
-PubSubConfig load_pubsub_config(JsonDocument doc, JsonSerializerOptions json_options) {
-   if (!doc.RootElement.TryGetProperty("PubSub", out var pubsub_element)) {
-      Log.Gen.warn("PubSub config section not found; using PubSubConfig.Default().");
-      return PubSubConfig.Default();
-   }
-
-   var pubsub_config = pubsub_element.Deserialize<PubSubConfig>(json_options);
-   if (pubsub_config is null) {
-      Log.Gen.error("PubSub config section could not be deserialized; using PubSubConfig.Default().");
-      return PubSubConfig.Default();
-   }
-
-   Log.Gen.info("Loaded PubSub config from DSPublisher config file.");
-   return pubsub_config;
-}
-
-DSLAPIServerConfig load_dsl_api_server_config(JsonDocument doc, JsonSerializerOptions json_options) {
-   if (!doc.RootElement.TryGetProperty("DSLAPIServer", out var dsl_api_server_element)) {
-      Log.Gen.warn("DSLAPIServer config section not found; using DSLAPIServerConfig defaults.");
-      return new DSLAPIServerConfig();
-   }
-
-   var dsl_api_server_config = dsl_api_server_element.Deserialize<DSLAPIServerConfig>(json_options);
-   if (dsl_api_server_config is null) {
-      Log.Gen.error("DSLAPIServer config section could not be deserialized; using DSLAPIServerConfig defaults.");
-      return new DSLAPIServerConfig();
-   }
-
-   Log.Gen.info($"Loaded DSLAPIServer config ip: {dsl_api_server_config.Ip} port: {dsl_api_server_config.Port}");
-   return dsl_api_server_config;
-}
-
-List<IDisposable> _process_publishing_nodes(IEnumerable<StreamPublishingNode> publishing_nodes, PubTableBroker pub_table_broker) {
-   var stream_publishers = new List<IDisposable>();
-   foreach (var node in publishing_nodes) {
-      if (string.IsNullOrWhiteSpace(node.Name) || string.IsNullOrWhiteSpace(node.Expression)) {
-         Log.Gen.error("Skipping StreamPublishingNode because name or expression is empty.");
-         continue;
-      }
-
-      Log.Gen.info($"Creating ViewPublisher name: {node.Name} expression: {node.Expression}");
-      var stream = new DSLStream(node.Expression);
-      stream_publishers.Add(new ViewPublisher(node.Name, pub_table_broker, stream));
-   }
-
-   return stream_publishers;
-}

src/DataStreamSuite/DSPublisher/Services/.gitkeep

@@ -1 +0,0 @@
-

src/DataStreamSuite/DSPublisher/WSPublishers/DatabentoPublisher.py

@@ -1,146 +0,0 @@
-"""
-Requires:
-   pip install -U databento
-   pip install websockets
-
-Sources:
-   https://databento.com/docs/quickstart/set-up
-   https://databento.com/docs/schemas-and-data-formats/bbo
-"""
-
-import datetime as dt
-import subprocess
-import sys
-import time
-
-try:
-   import databento as db
-except ImportError:
-   print("Installing missing dependency: databento")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
-   import databento as db
-
-from databento.common.error import BentoError
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-from databento_transformer import DatabentoTransformer
-from pubsub_client import PubSubClient
-
-
-API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
-DATASET = "GLBX.MDP3"
-SYMBOL = "ES.FUT"
-PUBSUB_WS_URL = "ws://localhost:19092/ws/"
-BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
-TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
-DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
-
-_pubsub_client = PubSubClient(PUBSUB_WS_URL)
-_transformer = DatabentoTransformer(
-   symbol=SYMBOL,
-   bbo_table_name=BBO_TABLE_NAME,
-   trades_table_name=TRADES_TABLE_NAME,
-   definitions_table_name=DEFINITIONS_TABLE_NAME,
-)
-
-
-def _publish_contract_definitions() -> None:
-   historical_client = db.Historical(API_KEY)
-   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
-   schema_range = dataset_range.get("schema", {}).get("definition", {})
-   available_end = schema_range.get("end", dataset_range["end"])
-
-   if isinstance(available_end, str):
-      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
-   else:
-      available_end_dt = available_end
-
-   # Databento's range end is exclusive. Back off slightly so the request stays
-   # strictly inside the available window.
-   end_dt = available_end_dt - dt.timedelta(seconds=1)
-   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
-   start = start_dt.isoformat()
-   end = end_dt.isoformat()
-
-   definition_data = historical_client.timeseries.get_range(
-      dataset=DATASET,
-      schema="definition",
-      stype_in="parent",
-      symbols=SYMBOL,
-      start=start,
-      end=end,
-   )
-
-   df = definition_data.to_df()
-   payload = _transformer.build_definitions_payload(df)
-   if payload is None:
-      print("No Databento contract definitions returned")
-      return
-
-   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
-   _pubsub_client.publish(payload)
-
-
-def on_record(record) -> None:
-   # Print every callback first so we can see whether Databento is flowing
-   # even if our row extraction logic misses a field shape.
-   print(record)
-
-   payloads = _transformer.transform_record(record)
-   if payloads:
-      for payload in payloads:
-         _pubsub_client.publish(payload)
-      return
-
-   print("Skipping record with no BBO/trade fields")
-
-
-def main() -> None:
-   _pubsub_client.start()
-   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
-   try:
-      _publish_contract_definitions()
-   except Exception as ex:
-      print(f"Contract definition publish failed: {ex}", flush=True)
-
-   while True:
-      try:
-         client = db.Live(key=API_KEY)
-
-         client.subscribe(
-            dataset=DATASET,
-            schema="bbo-1s",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-         client.subscribe(
-            dataset=DATASET,
-            schema="trades",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-
-         client.add_callback(on_record)
-         client.start()
-         print("Databento client started, waiting for BBO and trade records...", flush=True)
-         client.block_for_close()
-         print("Databento client closed cleanly", flush=True)
-         break
-      except BentoError as ex:
-         print(f"Databento live connection error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-      except Exception as ex:
-         print(f"Databento publisher unexpected error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-
-
-if __name__ == "__main__":
-   main()

src/DataStreamSuite/DSPublisher/WSPublishers/databento_transformer.py

@@ -1,244 +0,0 @@
-import datetime as dt
-
-
-class DatabentoTransformer:
-   def __init__(
-      self,
-      symbol: str,
-      bbo_table_name: str,
-      trades_table_name: str,
-      definitions_table_name: str,
-   ):
-      self._symbol = symbol
-      self._bbo_table_name = bbo_table_name
-      self._trades_table_name = trades_table_name
-      self._definitions_table_name = definitions_table_name
-      self._instrument_id_to_symbol: dict[int, str] = {}
-      self._last_bbo_row: dict | None = None
-      self._trade_row_id = 0
-      self._trade_table_created = False
-
-   def transform_record(self, record) -> list[dict]:
-      payload = self._handle_bbo_record(record)
-      if payload is not None:
-         return [payload]
-
-      payload = self._handle_trade_record(record)
-      if payload is not None:
-         return [payload]
-
-      return []
-
-   def build_definitions_payload(self, definition_df) -> dict | None:
-      if definition_df is None or definition_df.empty:
-         return None
-
-      df = definition_df.reset_index()
-      if "instrument_class" in df.columns:
-         df = df[df["instrument_class"].astype(str) == "F"]
-      if "security_type" in df.columns:
-         df = df[df["security_type"].astype(str) == "FUT"]
-      if "raw_symbol" in df.columns:
-         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
-         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
-
-      rows: list[dict] = []
-      for idx, row in enumerate(df.itertuples(index=False), start=1):
-         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
-         security_name = str(getattr(row, "raw_symbol", ""))
-         if instrument_id > 0 and security_name:
-            self._instrument_id_to_symbol[instrument_id] = security_name
-
-         rows.append({
-            "id": idx,
-            "security_name": security_name,
-            "instrument_id": instrument_id,
-            "instrument_class": str(getattr(row, "instrument_class", "")),
-            "security_type": str(getattr(row, "security_type", "")),
-            "asset": str(getattr(row, "asset", "")),
-            "exchange": str(getattr(row, "exchange", "")),
-            "currency": str(getattr(row, "currency", "")),
-            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
-            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
-            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
-            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
-            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
-            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
-            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
-            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
-            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
-            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
-            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
-            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
-            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
-            "underlying": str(getattr(row, "underlying", "")),
-         })
-
-      if not rows:
-         return None
-
-      return {
-         "op": "create_table",
-         "table_name": self._definitions_table_name,
-         "rows": rows,
-      }
-
-   def _handle_bbo_record(self, record) -> dict | None:
-      row = self._to_bbo_row(record)
-      if row is None:
-         return None
-
-      if self._last_bbo_row is None:
-         payload = {
-            "op": "create_table",
-            "table_name": self._bbo_table_name,
-            "rows": [row],
-         }
-      else:
-         payload = {
-            "op": "multiple_remove_add",
-            "table_name": self._bbo_table_name,
-            "removed_rows": [self._last_bbo_row],
-            "added_rows": [row],
-         }
-
-      self._last_bbo_row = row
-      return payload
-
-   def _handle_trade_record(self, record) -> dict | None:
-      row = self._to_trade_row(record)
-      if row is None:
-         return None
-
-      row["id"] = self._trade_row_id
-      self._trade_row_id += 1
-
-      if not self._trade_table_created:
-         payload = {
-            "op": "create_table",
-            "table_name": self._trades_table_name,
-            "rows": [row],
-         }
-         self._trade_table_created = True
-         return payload
-
-      return {
-         "op": "multiple_remove_add",
-         "table_name": self._trades_table_name,
-         "removed_rows": [],
-         "added_rows": [row],
-      }
-
-   def _to_bbo_row(self, record) -> dict | None:
-      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
-      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
-      if bid_px is None and ask_px is None:
-         return None
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "bid_px": float(bid_px or 0.0),
-         "ask_px": float(ask_px or 0.0),
-         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
-         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
-      }
-
-   def _to_trade_row(self, record) -> dict | None:
-      price = self._get_attr(record, "price", "px")
-      size = self._get_attr(record, "size")
-      if price is None and size is None:
-         return None
-
-      side = self._get_attr(record, "side", "aggressor_side", default="")
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "price": float(price or 0.0),
-         "size": int(size or 0),
-         "side": str(side),
-      }
-
-   def _get_security_name(self, record) -> str:
-      instrument_id = self._get_attr(record, "instrument_id", default=0)
-      try:
-         instrument_id_int = int(instrument_id or 0)
-      except (TypeError, ValueError):
-         instrument_id_int = 0
-
-      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
-         return self._instrument_id_to_symbol[instrument_id_int]
-
-      value = self._get_attr(
-         record,
-         "symbol",
-         "raw_symbol",
-         "stype_out_symbol",
-         "stype_in_symbol",
-         default=self._symbol,
-      )
-      return str(value)
-
-   def _get_timestamp(self, record) -> int:
-      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
-      try:
-         return int(value)
-      except (TypeError, ValueError):
-         return 0
-
-   def _get_attr(self, record, *names, default=None):
-      for name in names:
-         if hasattr(record, name):
-            value = getattr(record, name)
-            if value is not None:
-               return value
-      return default
-
-   def _get_level_attr(self, record, name, default=None):
-      levels = self._get_attr(record, "levels", default=None)
-      if levels:
-         level0 = levels[0]
-         if hasattr(level0, name):
-            value = getattr(level0, name)
-            if value is not None:
-               return value
-      return default
-
-   def _ns_to_iso(self, timestamp_ns) -> str:
-      if timestamp_ns is None:
-         return ""
-
-      if hasattr(timestamp_ns, "isoformat"):
-         try:
-            if hasattr(timestamp_ns, "to_pydatetime"):
-               timestamp_dt = timestamp_ns.to_pydatetime()
-            else:
-               timestamp_dt = timestamp_ns
-
-            if timestamp_dt is None:
-               return ""
-
-            if getattr(timestamp_dt, "tzinfo", None) is None:
-               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
-            return timestamp_dt.isoformat()
-         except (TypeError, ValueError, AttributeError):
-            pass
-
-      try:
-         ts_int = int(timestamp_ns)
-         if ts_int <= 0:
-            return ""
-         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
-      except (TypeError, ValueError, OSError, OverflowError):
-         return ""
-
-   def _nano_to_float(self, value) -> float:
-      try:
-         return float(value) / 1_000_000_000.0
-      except (TypeError, ValueError):
-         return 0.0

src/DataStreamSuite/DSPublisher/WSPublishers/pubsub_client.py

@@ -1,58 +0,0 @@
-import asyncio
-import json
-import queue
-import threading
-
-import websockets
-
-
-class PubSubClient:
-   def __init__(self, ws_url: str):
-      self._ws_url = ws_url
-      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
-      self._sender_thread: threading.Thread | None = None
-
-   def start(self) -> threading.Thread:
-      if self._sender_thread is None or not self._sender_thread.is_alive():
-         self._sender_thread = self._start_sender_thread()
-      return self._sender_thread
-
-   def publish(self, payload: dict) -> None:
-      self._outgoing_queue.put(payload)
-
-   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
-      self.publish({
-         "op": "create_table",
-         "table_name": table_name,
-         "rows": rows,
-      })
-
-   def publish_multiple_remove_add(
-      self,
-      table_name: str,
-      removed_rows: list[dict],
-      added_rows: list[dict],
-   ) -> None:
-      self.publish({
-         "op": "multiple_remove_add",
-         "table_name": table_name,
-         "removed_rows": removed_rows,
-         "added_rows": added_rows,
-      })
-
-   def _start_sender_thread(self) -> threading.Thread:
-      thread = threading.Thread(target=self._run_send_loop, daemon=True)
-      thread.start()
-      return thread
-
-   def _run_send_loop(self) -> None:
-      asyncio.run(self._ws_send_loop())
-
-   async def _ws_send_loop(self) -> None:
-      async with websockets.connect(self._ws_url) as ws:
-         print(f"Connected to PubSubServer websocket: {self._ws_url}")
-         while True:
-            payload = await asyncio.to_thread(self._outgoing_queue.get)
-            body = json.dumps(payload)
-            print(body)
-            await ws.send(body)

src/DataStreamSuite/DSPublisher/appsettings.Development.json

@@ -1,69 +0,0 @@
-{
-  "PubSub": {
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "DisposedConnectionCheckPeriod": 1000,
-    "ServerPort": 22019,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  },
-  "DSLAPIServer": {
-    "Ip": "localhost",
-    "Port": 19093
-  },
-  "DSPublisher": {
-    "StreamPublishingNodes": [
-      {
-        "Name": "sample_stream",
-        "Expression": "MillisecondStream(1000,5)"
-      }
-    ],
-    "PythonPublishingNodes": [
-      {
-        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
-        "Args": ""
-      }
-    ]
-  }
-}

src/DataStreamSuite/DSPublisher/appsettings.json

@@ -1,69 +0,0 @@
-{
-  "PubSub": {
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "DisposedConnectionCheckPeriod": 1000,
-    "ServerPort": 22019,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  },
-  "DSLAPIServer": {
-    "Ip": "localhost",
-    "Port": 19093
-  },
-  "DSPublisher": {
-    "StreamPublishingNodes": [
-      {
-        "Name": "sample_stream",
-        "Expression": "MillisecondStream(1000,5)"
-      }
-    ],
-    "PythonPublishingNodes": [
-      {
-        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
-        "Args": ""
-      }
-    ]
-  }
-}

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -2,8 +2,7 @@
    using BCG;
 
    [DataStreamUserRespresentation("AddColumnsStream")]
-   // example: AddColumnsStream(l,"jc1:001")
-   public class AddColumnsStream : ViewStream {
+   public class AddColumnsStream: ViewStream {
       public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
          ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/HideColumnsStream.cs

@@ -1,126 +0,0 @@
-namespace BlueOrb.Operators.Core {
-   using BCG;
-
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-
-   [DataStreamUserRespresentation("HideColumnsStream")]
-   public class HideColumnsStream : ViewStream {
-      // Wrapper row to hide the columns
-      public class MyRow : IBlueRow {
-         #region IBlueRow members
-         private IBlueTableSchema _schema;
-         private IBlueRow _row;
-         public MyRow(IBlueTableSchema schema, IBlueRow row) {
-            _schema = schema;
-            _row = row;
-         }
-         public IBlueTableSchema schema { get { return _schema; } }
-         #endregion
-
-         #region ICollection members
-         public int Count { get { return schema.columnNames.Length; } }
-
-         public bool IsSynchronized { get { return false; } }
-
-         public object SyncRoot { get { return this; } }
-
-         public void CopyTo(Array array, int index) {
-            int i = index;
-            foreach (object o in this as IEnumerable)
-               array.SetValue(o, i++);
-         }
-         #endregion
-
-         #region IDictionary members
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public object this[object key] {
-            get {
-               if (key is string)
-                  return this[(string)key];
-               return this[(int)key];
-            }
-            set { throw new NotSupportedException("The IBlueRow is readonly"); }
-         }
-
-         public object this[int index] {
-            get {
-               return this[schema.columnInfo[index].colName];
-            }
-         }
-
-         public object this[string col_name] {
-            get {
-               if (_schema.contains_column(col_name))
-                  return _row[col_name];
-               else
-                  throw new ApplicationException($"{col_name} not found");
-            }
-         }
-
-         public ICollection Keys { get { return schema.columnNames; } }
-
-         public ICollection Values {
-            get {
-               object[] retVal = new object[schema.columnNames.Length];
-               CopyTo(retVal, 0);
-               return retVal;
-            }
-         }
-
-         public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-         public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-         public bool Contains(object key) { return schema[(string)key] != null; }
-
-         public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-         #endregion
-
-      }
-
-      private readonly ViewStream _parent_str;
-      private readonly HashSet<string> _removed_cols;
-      private BlueTable _state;
-      public HideColumnsStream(ViewStream parent_str, string hide_cols_expr) {
-         _parent_str = parent_str;
-         _removed_cols = new HashSet<string>(hide_cols_expr.Split(',', ' '));
-         if (_removed_cols.Contains("id"))
-            _removed_cols.Remove("id");
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _handle_parent_str_cb;
-      }
-
-      private void _handle_parent_str_cb(object dp_obj) {
-         // this could be made incremental and hence faster
-         DataPacket dp_in = dp_obj as DataPacket;
-         if (_state == null) {
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            foreach (ColumnInfo ci in dp_in.schema.columnInfo) {
-               if (!_removed_cols.Contains(ci.colName))
-                  cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
-            }
-            _state = new BlueTable(new BlueTableSchema(dp_in.schema.tableName, cis.ToArray()));
-         }
-         _state.clear();
-         foreach (IBlueRow r in dp_in.newRows) {
-            _state.addRow(new MyRow(_state.schema, r));
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_state,
-            myLastValue == null ? new BlueTable(_state.schema) : (myLastValue as DataPacket).newRows);
-         myFireEvent(dp_out);
-      }
-   }
-
-
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/CartesianProductStream.cs

@@ -1,14 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("CartesianProductStream")]
-   public class CartesianProductStream : ViewStream {
-      public CartesianProductStream(ViewStream ls, ViewStream rs) {
-         ls = new AddColumnsStream(ls, "ljc: 001");
-         rs = new AddColumnsStream(rs, "rjc: 001");
-         ViewStream res = new SimpleJoinStream(ls, "ljc", rs, "rjc");
-         res = new global::BlueOrb.Operators.Core.HideColumnsStream(res, "ljc, rjc");
-         mySetRealSource(res);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/FullJoinStream.cs

@@ -1,15 +0,0 @@
-namespace BlueOrb.Operators.Core {
-   using System;
-
-   using BCG;
-
-   [DataStreamUserRespresentation("FullJoinStream")]
-   // JoinStream is technically a LeftJoin. FullJoin only results in rows that have bpth left and right side
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new global::bts.BlueOrb.SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,5 +1,4 @@
-namespace bts.BlueOrb
-{
+namespace bts.BlueOrb {
    using BCG;
 
    using Bts.DataStream;
@@ -14,10 +13,8 @@
    using RowHash = System.Int32;
 
    [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream
-   {
-      public class JoinBookkeepingData
-      {
+   public class SimpleJoinStream : ViewStream {
+      public class JoinBookkeepingData {
          // initialized in cons
          readonly string[] _join_col_names;
          private readonly BlueRowHasher _hasher;
@@ -26,8 +23,7 @@
          int[]? _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
 
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher)
-         {
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
             _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
             _hasher = row_hasher;
          }
@@ -44,28 +40,22 @@
          public List<IBlueRow>? get_matching_rows(int hash) =>
             _hash_row_map.GetValueOrDefault(hash);
 
-         private RowHash _get_hash(IBlueRow r)
-         {
-            if (_join_cols == null || _join_cols.Length == 0)
-            {
+         private RowHash _get_hash(IBlueRow r) {
+            if (_join_cols == null || _join_cols.Length == 0) {
                return 0;
             }
             return _hasher.get_hash(r, _join_cols);
          }
 
-         public void update_data(IDataPacket dp)
-         {
-            if (!ready)
-            {
+         public void update_data(IDataPacket dp) {
+            if (!ready) {
                _schema = dp.newRows.schema;
                _join_cols = _schema.ResolveIndices(_join_col_names);
             }
             _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows)
-            {
+            foreach (IBlueRow r in dp.newRows) {
                int hash = _get_hash(r);
-               if (!_hash_row_map.TryGetValue(hash, out var hash_rows))
-               {
+               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
                   hash_rows = new List<IBlueRow>();
                   _hash_row_map.Add(hash, hash_rows);
                }
@@ -74,10 +64,8 @@
          }
 
          [MemberNotNullWhen(true, nameof(_schema))]
-         public bool ready
-         {
-            get
-            {
+         public bool ready {
+            get {
                return _schema != null;
             }
          }
@@ -89,11 +77,9 @@
       IBlueTableSchema? _schema;
 
       public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc)
-      { }
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
 
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc)
-      {
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
          _ls = ls;
          _rs = rs;
          List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
@@ -105,35 +91,30 @@
          _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          _ls.onChange += _ls_onchange;
          _rs.onChange += _rs_onchange;
       }
 
-      private void _ls_onchange(object dp_obj)
-      {
+      private void _ls_onchange(object dp_obj) {
          var dp = (IDataPacket)dp_obj;
          _ldata.update_data(dp);
          _form_and_fire();
       }
 
-      private void _rs_onchange(object dp_obj)
-      {
+      private void _rs_onchange(object dp_obj) {
          var dp = (IDataPacket)dp_obj;
          _rdata.update_data(dp);
          _form_and_fire();
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          _ls.onChange -= _ls_onchange;
          _rs.onChange -= _rs_onchange;
       }
 
       [MemberNotNull(nameof(_schema))]
-      private void _build_schema()
-      {
+      private void _build_schema() {
          if (_schema != null)
             return;
          var lschema = Log.notnull(_ldata.schema);
@@ -141,64 +122,52 @@
          var cis = new List<ColumnInfo> {
             new("id", 0, TypeCode.Int32)
          };
-         foreach (ColumnInfo ci in lschema.columnInfo)
-         {
+         foreach (ColumnInfo ci in lschema.columnInfo) {
             string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         foreach (ColumnInfo ci in rschema.columnInfo)
-         {
+         foreach (ColumnInfo ci in rschema.columnInfo) {
             string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
          _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
       }
 
-      private void _form_and_fire()
-      {
+      private void _form_and_fire() {
          if (!_ldata.ready || !_rdata.ready)
             return;
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
 
          // this is where the join happens
-         foreach (var kv in _ldata.rows)
-         {
+         foreach (var kv in _ldata.rows) {
             RowHash hash = kv.Key;
             List<IBlueRow> l_rows = kv.Value;
             List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
 
-            foreach (IBlueRow lr in l_rows)
-            {
-               if (r_rows != null)
-               {
-                  foreach (IBlueRow r in r_rows)
-                  {
+            foreach (IBlueRow lr in l_rows) {
+               if (r_rows != null) {
+                  foreach (IBlueRow r in r_rows) {
                      object[] rowvals = build_row_array(res_rows.Count, lr);
                      copy_row(rowvals, r, false);
                      res_rows.Add(rowvals);
                   }
-               }
-               else
-               {
+               } else {
                   object[] rowvals = build_row_array(res_rows.Count, lr);
                   res_rows.Add(rowvals);
                }
             }
          }
          BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows)
-         {
+         foreach (object[] rval in res_rows) {
             rv.addRow(new BlueRow(_schema, rval));
          }
          var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
          myFireEvent(dp);
 
-         object[] build_row_array(int id, IBlueRow l_row)
-         {
+         object[] build_row_array(int id, IBlueRow l_row) {
             object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++)
-            {
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
                rv[i] = DBNull.Value;
             }
             rv[0] = id;
@@ -206,26 +175,30 @@
             return rv;
          }
 
-         void copy_row(object?[] rval, IBlueRow r, bool is_left)
-         {
+         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
             int offset = 1;
             JoinBookkeepingData sdata = _ldata;
-            if (is_left)
-            {
+            if (is_left) {
                offset = 1;
                sdata = _ldata;
-            }
-            else
-            {
+            } else {
                sdata = _rdata;
                offset = 1 + _ldata.RequireSchema().columnInfo.Length;
             }
-            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++)
-            {
+            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
                rval[offset + i] = r[i];
             }
          }
       }
    }
 
+   [DataStreamUserRespresentation("FullJoinStream")]
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+
 }

src/DataStreamSuite/DataStream/BlueOrb/TestStream.cs

@@ -0,0 +1,58 @@
+namespace BlueOrb.Internal;
+
+using BCG;
+
+using System;
+using System.Linq;
+
+[DataStreamUserRespresentation("TestStream")]
+public class TestStream : ViewStream {
+   public TestStream() {
+      var rs = new BlueOrb.Operators.Core.LiteralStream("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc");
+      mySetRealSource(rs);
+   }
+}
+
+[DataStreamUserRespresentation("OpsStream")]
+public class OpsStream : ViewStream {
+   public OpsStream() {
+   }
+
+   protected override void myOnFirstSubscriber() {
+      IBlueTableSchema schema = new BlueTableSchema("Ops");
+      schema = schema.add_column(System.TypeCode.Int32, "id");
+      schema = schema.add_column(System.TypeCode.String, "op");
+      schema = schema.add_column(System.TypeCode.String, "namespace");
+      schema = schema.add_column(System.TypeCode.String, "assembly_name");
+      schema = schema.add_column(System.TypeCode.String, "constructor");
+      var loaded_streams = PrimitiveManager.instance.get_loaded_streams();
+      BlueTable rv = new BlueTable(schema);
+      foreach (var kv in loaded_streams.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase)) {
+         string sn = kv.Key;
+         Type t = kv.Value;
+         string ns = t.Namespace ?? string.Empty;
+         string assembly_name = t.Assembly.GetName().Name ?? string.Empty;
+         var constructors = t.GetConstructors()
+            .OrderBy(c => c.GetParameters().Length)
+            .ThenBy(c => c.ToString());
+         if (!constructors.Any()) {
+            IBlueRow r = new BlueRow(schema, rv.Count, sn, ns, assembly_name, "(no public constructor)");
+            rv.addRow(r);
+            continue;
+         }
+         foreach (var ctor in constructors) {
+            var parameters = ctor.GetParameters();
+            string constructor = $"{t.Name}({string.Join(", ", parameters.Select(p => p.ParameterType.Name))})";
+            IBlueRow r = new BlueRow(schema, rv.Count, sn, ns, assembly_name, constructor);
+            rv.addRow(r);
+         }
+      }
+      myFireEvent(new DataPacket(rv, rv, new BlueTable(schema)));
+   }
+
+   protected override void myOnLastSubscriber() {
+      base.myOnLastSubscriber();
+   }
+
+
+}

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -29,26 +29,22 @@ namespace bts.BlueOrb {
          _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
-      private bool _orig_schema_has_timestamp = false;
+
       private void _onstream_change(object o) {
          DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
          if (_table == null) {
             IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            if (!schema.contains_column("timestamp")) {
-               _orig_schema_has_timestamp = true;
-               schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
-            }
+            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
             _table = new CircularBufferTable(schema, _size);
          }
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            if (!_orig_schema_has_timestamp)
-               ar.setValue("timestamp", now);
+            ar.setValue("timestamp", now);
             ar.setValue("id", _id++);
             _table.add_row(ar);
          }
-         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         var dp_out =  BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
          myFireEvent(dp_out);
       }
    }

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -23,16 +23,16 @@ public static class DataStreamExtensions {
    public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
       source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
 
-   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
 
-      IDisposable Subscribe(IObserver<DataPacket> observer) {
+      IDisposable Subscribe(IObserver<IDataPacket> observer) {
          source.onChange += OnChange;
          return Disposable.Create(() => source.onChange -= OnChange);
 
          void OnChange(object? newValue) =>
-            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
-               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
+            observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
+               $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
 

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -88,8 +88,7 @@ namespace BCG {
          return Task.CompletedTask;
       }
 
-      private void _on_str(DataPacket dp)
-      {
+      void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");

src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs

@@ -0,0 +1,22 @@
+// using System;
+// using BCG;
+// using BCG.Utilities;
+
+// namespace BCG
+// {
+//    public class CartesianProductStream : ViewStream
+//    {
+//       public CartesianProductStream(ViewStream v1, ViewStream v2)
+//       {
+//          mySetRealSource(
+//          new JoinStream (new AddColumnsStream (v1,
+//             "hiddenCartesianColumn=1"),
+//             "hiddenCartesianColumn",
+//             "!id hiddenCartesianColumn dateTime:dateBoundary",
+//             new AddColumnsStream (v2,
+//             "hiddenCartesianColumn=1"),
+//             "hiddenCartesianColumn",
+//             "!id hiddenCartesianColumn"));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs

@@ -14,10 +14,8 @@ namespace BCG
 		public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
          : base (colMapExpression, null, columnNames) {}
 
-
-      public override IBlueRow evaluate (DataPacket packet)
-      {
-         myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
+      public override IBlueRow? evaluate(IDataPacket packet) {
+         myOutputTable ??= new GroupByOutputTable(packet.Schema);
          string hashString = GroupByStream.getRepString (packet, myGroupByCols);
          if (packet.newRows.Count == 0)
             //if there are no rows for this group we remove the group from the output table

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -42,7 +42,7 @@ namespace BCG
          return retVal;
       }
 
-      public abstract IBlueRow evaluate (DataPacket dp) ;
+      public abstract IBlueRow? evaluate(IDataPacket dp);
       #endregion
 
       #region Helper methods

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -8,12 +8,13 @@ namespace BCG {
    internal class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
-         private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
+         readonly Func<IDataPacket, IBlueRow> _groupby_func;
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
             _groupby_func = groupby_func;
          }
-         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
+         public IBlueRow evaluate(IDataPacket packet) =>
+            _groupby_func(packet);
 
          public string[] groupByColumns { get { return _groupby_cols; } }
 
@@ -32,7 +33,7 @@ namespace BCG {
       private IBlueTableSchema myOutputSchema;
       private int myIdGenerator = 0;
 
-      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
+      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
          : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
       public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
@@ -187,11 +188,11 @@ namespace BCG {
          }
       }
 
-      public static string getRepString(DataPacket dp, string[] colNames) {
+      public static string getRepString(IDataPacket dp, string[] colNames) {
          return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow(DataPacket dp) {
+      public static IBlueRow getRepRow(IDataPacket dp) {
          if (dp.newRows.Count > 0)
             return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)

src/DataStreamSuite/DataStream/Primitives/GroupBy/IGroupByEvaluator.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG
 {
    /// <summary>
@@ -14,7 +12,7 @@ namespace BCG
       /// deletetion) and the added and deletedRows being the increments. If a group has
       /// to be totally removed then a null should be returned
       /// </summary>
-      IBlueRow evaluate (DataPacket packet);
+      IBlueRow? evaluate(IDataPacket packet);
 
       /// <summary>
       /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -14,9 +14,8 @@ namespace BCG {
         public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMaxRow = myOutputTable.getRow(hashString);
             IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,7 +1,3 @@
-using System;
-
-using BCG.Utilities;
-
 namespace BCG {
     public class MinGroupByEvaluator : MaxGroupByEvaluator {
         public MinGroupByEvaluator(string outputCol, string[] groupByCols)
@@ -10,8 +6,8 @@ namespace BCG {
         public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
            base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMinRow = myOutputTable.getRow(stringRep);
             IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG {
     /// <summary>
     /// Computes the sum for a given column
@@ -10,8 +8,8 @@ namespace BCG {
         public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
 
             IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
             return sumRow;
@@ -33,7 +31,7 @@ namespace BCG {
 
         #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
+        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);
 
         public string[] groupByColumns { get { return myGroupByCols; } }
 
@@ -51,9 +49,8 @@ namespace BCG {
         public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-
+        public override IBlueRow evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
             return sumRow;
         }
@@ -68,7 +65,7 @@ namespace BCG {
 
         #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
+        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);
 
         public string[] groupByColumns { get { return myGroupByCols; } }
 

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -2,6 +2,8 @@ using BCG;
 
 using MongoDB.Driver.Core.WireProtocol.Messages;
 
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -37,7 +39,7 @@ namespace bts {
       }
 
       void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj);
          IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
          BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
          :

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -8,34 +8,48 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using bts;
-using BCG;
 
-namespace BlueOrb.Pubsub.Diagnostics {
-   // This publishes data to the pubsub server, subscribes, gets it back and records the time it took for the round trip
+namespace bts
+{
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker {
-      public class PayloadMessage {
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
          public readonly string? msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[]? payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte) {
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++) {
+            for (int i = 0; i < payload_size; i++)
+            {
                rv[i] = payload_byte;
             }
             return rv;
          }
 
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
+         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
+         // {
+         //    guid = guid_;
+         //    timestamp = timestamp_;
+         //    payload = new byte[payload_size];
+         //    for (int i = 0; i < payload_size; i++)
+         //    {
+         //       payload[i] = payload_byte;
+         //    }
+         // }
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize() {
+         public byte[] serialize()
+         {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id ?? string.Empty);
@@ -47,7 +61,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes) {
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
             var str = new MemoryStream(ser_bytes) {
                Position = 0
             };
@@ -72,7 +87,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState {
+      private enum IOState
+      {
          SEND = 0,
          RECV = 1
       };
@@ -107,7 +123,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _pub_core = PublisherCore.Create(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -115,7 +132,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
 
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _vsd.removeWorker(this);
          Log.notnull(_sub_core).unsubscribe(_tag_name);
       }
@@ -126,7 +144,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
             SubscriberRoutes = new() { new(serverUrl, ".*") },
          });
 
-      private void _fire_dp_after_send() {
+      private void _fire_dp_after_send()
+      {
          Log.notnull(_last_sent_msg);
          // as soon as the data is published, its reflected in the stream state
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -136,7 +155,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv() {
+      private void _fire_dp_after_recv()
+      {
          Log.notnull(_last_recvd_msg);
          Log.notnull(_last_sent_msg);
          // update the last publish row
@@ -153,11 +173,13 @@ namespace BlueOrb.Pubsub.Diagnostics {
          }
       }
 
-      private string _get_next_id() {
+      private string _get_next_id()
+      {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet() {
+      private void _publish_next_packet()
+      {
          Log.notnull(_pub_core);
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
@@ -166,10 +188,13 @@ namespace BlueOrb.Pubsub.Diagnostics {
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork() {
-         switch (_state) {
+      public void doWork()
+      {
+         switch (_state)
+         {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period) {
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -181,7 +206,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
                var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null) {
+               if (recv_msg != null)
+               {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -204,7 +230,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
          }
       }
 
-      public static long epoch_now() {
+      public static long epoch_now()
+      {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -1,17 +1,12 @@
 namespace bts {
    using BCG;
 
-   using bts.collections;
-
-   using IronPython.Runtime.Operations;
-
    using ProtoBuf;
 
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
-   using System.Runtime.Intrinsics.Arm;
 
    public class ProjectData {
       // schema from the base stream

src/DataStreamSuite/DataStream/Publish/PublisherStream.cs

@@ -1,5 +1,4 @@
-namespace BlueOrb.PubSub {
-   using BCG;
+namespace BCG {
    // this for testing purposes mainly, its wraps a ViewPublisher around a stream, 
    // and hence epublishes it out to the PS
    public class PublisherStream : ViewStream {

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -15,7 +15,6 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
 using bts;
@@ -360,11 +359,11 @@ namespace BCG {
       }
 
       public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
-         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
+         Dictionary<string, string> dictionary = JsonSerializer.Deserialize<Dictionary<string, string>>(json_string);
          return dictionary;
       }
       public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
-         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
+         Dictionary<string, object> dictionary = JsonSerializer.Deserialize<Dictionary<string, object>>(json_string);
          if (!dictionary.ContainsKey("id")) {
             dictionary.Add("id", id);
          }
@@ -1847,7 +1846,7 @@ namespace BCG {
          return sb.ToString();
       }
 
-      public static string rows_id_string(this IBlueRowCollection table) {
+      public static string id_string(IBlueRowCollection table) {
          string rv = String.Empty;
          foreach (IBlueRow r in table)
             rv += $"{r["id"]} ";
@@ -2246,7 +2245,7 @@ namespace BCG {
 
       public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
          try {
-            object rv = (r != null && r.schema.contains_column(col_name)) ? r[col_name] : DBNull.Value;
+            object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
             if (rv == null || rv == DBNull.Value)
                return error_val;
             else

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,4 +1,6 @@
 #nullable enable
+using Bts.DataStreamCore;
+
 using System;
 using System.Text.Json.Serialization;
 
@@ -7,6 +9,7 @@ namespace BCG {
    /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
    /// </summary>
    [Serializable]
+   [JsonConverter(typeof(ColumnInfoJsonConverter))]
    public class ColumnInfo : IComparable<ColumnInfo>, IComparable, IEquatable<ColumnInfo> {
 
       //the default column info does not cach values in a lookup table, therefore it is in group 0
@@ -67,16 +70,12 @@ namespace BCG {
       private Type myColType;
       private object? myColValue;
 
-      [JsonPropertyName("colNo")]
       public int colNo { get { return myColNo; } }
 
-      [JsonPropertyName("colName")]
       public string colName { get { return myColName; } }
 
-      [JsonPropertyName("colType")]
       public Type colType { get { return myColType; } }
 
-      [JsonIgnore]
       public object? colValue {get{return myColValue;}}
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfoJsonConverter.cs

@@ -0,0 +1,27 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class ColumnInfoJsonConverter : JsonConverter<ColumnInfo> {
+   public override ColumnInfo Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      var jobj = JsonSerializer.Deserialize<JsonObject>(ref reader);
+      return new(
+         jobj[nameof(ColumnInfo.colName)].GetValue<string>(),
+         jobj[nameof(ColumnInfo.colNo)].GetValue<int>(),
+         Type.GetType(jobj[nameof(ColumnInfo.colType)].GetValue<string>()));
+   }
+
+   public override void Write(Utf8JsonWriter writer, ColumnInfo value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      writer.WriteString(nameof(ColumnInfo.colName), value.colName);
+      writer.WriteNumber(nameof(ColumnInfo.colNo), value.colNo);
+      writer.WriteString(nameof(ColumnInfo.colType), value.colType.ToString());
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ExtensionFunctions/Comparisons.cs

@@ -1,142 +0,0 @@
-namespace BlueOrb.Data {
-   using BCG;
-
-   using bts;
-
-
-   using System;
-   using System.Linq;
-
-   public static class Comparisons {
-      public static int compare_row(this IBlueRow row, IBlueRow other_row, params string[] col_names) {
-         int comp_res = 0;
-         foreach (string col_name in col_names) {
-            object val1 = row[col_name];
-            object val2 = other_row[col_name];
-            comp_res = compare_row_values(val1, val2);
-            if (comp_res != 0)
-               break;
-         }
-         return comp_res;
-      }
-
-      public static void compare_tables_id(this IBlueRowCollection table1,
-                                           IBlueRowCollection table2,
-                                           out IBlueRowCollection rows_only_in_table1,
-                                           out IBlueRowCollection rows_only_in_table2) {
-         Log.assert(table1 != null);
-         table2 = table2 ?? new BlueTable(table1.schema);
-         var schema = get_output_schema(table1, table2);
-         var table1_only = new BlueTable(schema);
-         var table2_only = new BlueTable(schema);
-
-         if (table1 is not null) {
-            foreach (IBlueRow row in table1)
-               if (table2 is null || !table2.containsID(row.get<int>("id")))
-                  table1_only.addRow(row);
-         }
-
-         if (table2 is not null) {
-            foreach (IBlueRow row in table2)
-               if (table1 is null || !table1.containsID(row.get<int>("id")))
-                  table2_only.addRow(row);
-         }
-
-         rows_only_in_table1 = table1_only;
-         rows_only_in_table2 = table2_only;
-      }
-
-      public static void compare_tables_deep(this IBlueRowCollection table1,
-                                             IBlueRowCollection table2,
-                                             out IBlueRowCollection rows_only_in_table1,
-                                             out IBlueRowCollection rows_only_in_table2) {
-         Log.assert(table1 != null);
-         table2 = table2 ?? new BlueTable(table1.schema);
-         var schema = get_output_schema(table1, table2);
-         validate_matching_schemas(table1, table2);
-
-         var table1_only = new BlueTable(schema);
-         var table2_only = new BlueTable(schema);
-         var compare_cols = schema.columnNames.Where(c => c != "id").ToArray();
-
-         if (table1 is not null) {
-            foreach (IBlueRow row in table1) {
-               if (table2 is null || !table2.containsID(row.get<int>("id"))) {
-                  table1_only.addRow(row);
-                  continue;
-               }
-
-               var other_row = table2[row.get<int>("id")];
-               if (row.compare_row(other_row, compare_cols) != 0) {
-                  table1_only.addRow(row);
-                  table2_only.addRow(other_row);
-               }
-            }
-         }
-
-         if (table2 is not null) {
-            foreach (IBlueRow row in table2)
-               if (table1 is null || !table1.containsID(row.get<int>("id")))
-                  table2_only.addRow(row);
-         }
-
-         rows_only_in_table1 = table1_only;
-         rows_only_in_table2 = table2_only;
-      }
-
-      static int compare_row_values(object v1, object v2) {
-         if (is_null(v1))
-            return is_null(v2) ? 0 : -1;
-         if (is_null(v2))
-            return 1;
-         try {
-            var comparable1 = (IComparable)v1;
-            return comparable1.CompareTo(v2);
-         } catch (InvalidCastException) {
-            if (v1 is byte[] bytes1 && v2 is byte[] bytes2)
-               return compare_bytes(bytes1, bytes2) ? 0 : 1;
-            throw;
-         }
-      }
-
-      static bool compare_bytes(byte[] v1, byte[] v2) {
-         if (v1.Length != v2.Length)
-            return false;
-
-         for (int i = 0; i < v1.Length; i++)
-            if (v1[i] != v2[i])
-               return false;
-
-         return true;
-      }
-
-      static bool is_null(object o) {
-         return o == null || o == DBNull.Value;
-      }
-
-      static IBlueTableSchema get_output_schema(IBlueRowCollection table1, IBlueRowCollection table2) {
-         if (table1 is not null)
-            return table1.schema;
-         if (table2 is not null)
-            return table2.schema;
-         throw new ArgumentNullException(nameof(table1), "At least one table must be non-null.");
-      }
-
-      static void validate_matching_schemas(IBlueRowCollection table1, IBlueRowCollection table2) {
-         if (table1 is null || table2 is null)
-            return;
-
-         var schema1 = table1.schema;
-         var schema2 = table2.schema;
-         if (schema1.columnInfo.Length != schema2.columnInfo.Length)
-            throw new ArgumentException("Tables must have matching schemas.");
-
-         for (int i = 0; i < schema1.columnInfo.Length; i++) {
-            var c1 = schema1.columnInfo[i];
-            var c2 = schema2.columnInfo[i];
-            if (c1.colName != c2.colName || c1.colType != c2.colType)
-               throw new ArgumentException("Tables must have matching schemas.");
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/ExtensionFunctions/ToStrings.cs

@@ -1,35 +0,0 @@
-namespace BlueOrb.Data {
-   using BCG;
-
-   using System;
-   using System.Text;
-
-public static class ToStrings {
-      public static string ToDebugString(this IBlueRow row) {
-         StringBuilder sb = new StringBuilder();
-         foreach (ColumnInfo c in row.schema.columnInfo) {
-            if (sb.Length != 0)
-               sb.Append(", ");
-            sb.Append(c.colName);
-            sb.Append(": ");
-            object val = row[c.colName];
-            sb.Append(is_null(val) ? "NULL" : val.ToString());
-         }
-         return sb.ToString();
-      }
-
-      public static string ToDebugString(this IBlueRowCollection rows) {
-         StringBuilder sb = new StringBuilder();
-         foreach (IBlueRow row in rows) {
-            if (sb.Length != 0)
-               sb.AppendLine();
-            sb.Append(row.ToDebugString());
-         }
-         return sb.ToString();
-      }
-
-      static bool is_null(object o) {
-         return o == null || o == DBNull.Value;
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -41,7 +41,7 @@ namespace BCG {
       }
 
       public IBlueRowCollection newRows, addedRows, deletedRows;
-      public IDictionary<string, object> meta_data = null;
+      public Dictionary<string, object> meta_data = null;
       public DataPacket() { }
 
       public DataPacket(IBlueTableSchema schema)
@@ -53,7 +53,7 @@ namespace BCG {
          this.deletedRows = deletedRows;
       }
       public DataPacket(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows,
-                  IDictionary<string, object> meta_data) {
+                  Dictionary<string, object> meta_data) {
          this.newRows = newRows;
          this.addedRows = addedRows;
          this.deletedRows = deletedRows;
@@ -63,6 +63,7 @@ namespace BCG {
       IBlueRowCollection IDataPacket.newRows => newRows;
       IBlueRowCollection IDataPacket.addedRows => addedRows;
       IBlueRowCollection IDataPacket.deletedRows => deletedRows;
+      IReadOnlyDictionary<string, object> IDataPacket.Metadata => meta_data;
 
       public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
          new_rows = newRows;

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -8,7 +8,6 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="OpenAI" Version="2.0.0" />
     <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
   </ItemGroup>

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -6,89 +6,89 @@ namespace BCG {
    using System.Reflection;
    using System.IO;
    using Bts.DataStreamCore;
-   using System.Collections.Generic;
-   using System.Linq;
 
    public class PrimitiveManager {
       static MarketDataStreamFactory mdStreamFactory;
-      private Dictionary<string, Type> _dslopname_to_type_map = new Dictionary<string, Type>();
-      private static PrimitiveManager _self;
-      private HashSet<string> _loaded_assemblies = new HashSet<string>();
-      private static string[] _allowed_namespaces_matches = new string[] {
-         "blueorb"
-      };
+      private IDictionary myTypeLookupTable = new Hashtable();
+      private static PrimitiveManager mySelf;
+      private ArrayList myAssemblyNames = new ArrayList();
 
       private PrimitiveManager() {
-         _load_all_streams();
+         myLoadAllTypes();
       }
 
-      private static bool _is_loadable_stream(Type type) {
-         Func<bool> namespace_match = () => {
-            foreach (string ns in _allowed_namespaces_matches) {
-               if (type.Namespace.ToLower().Contains(ns.ToLower()))
-                  return true;
-            }
-            return false;
-         };
-         bool rv = type.IsSubclassOf(typeof(ViewStream))
-                   && type.IsPublic;
-         //rv = rv && (!type.IsInterface && !type.IsAbstract);
-         rv = rv && namespace_match();
-         return rv;
-      }
-
-      private void _load_all_streams() {
-         load_streams_from_assembly(this.GetType().Assembly.FullName);
+      private void myLoadAllTypes() {
+         AppDomain currentDomain = AppDomain.CurrentDomain;
+         currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
+         loadAssembly(this.GetType().Assembly.FullName);
          var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
-         foreach (string assembly_name in assemblies) {
-            bts.Log.Gen.info($"PrimitiveManager._load_all_streams assemblyName: {assembly_name}");
-            string resolved_path_name = assembly_name;
-            if (!System.IO.Path.IsPathRooted(assembly_name)) {
-               resolved_path_name = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assembly_name));
+         foreach (string assemblyName in assemblies) {
+            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
+            string resolvedAssemblyPath = assemblyName;
+            if (!System.IO.Path.IsPathRooted(assemblyName)) {
+               resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
             }
-            load_streams_from_assembly(resolved_path_name);
+            loadAssembly(resolvedAssemblyPath);
          }
       }
 
-      public Type get_dsl_op_type(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            return null;
-         return rv;
+      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
+         foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
+            if (args.Name.Equals(a.FullName))
+               return a;
+         return null;
       }
 
-      public void load_streams_from_assembly(string assembly_name) {
-         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assembly_name}");
-         if (_loaded_assemblies.Contains(assembly_name))
+      /// <summary>
+      /// Loads all the Stream types associated with the assembly
+      /// </summary>
+      /// <param name="assemblyName"></param>
+      public void loadAssembly(string assemblyName) {
+         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
+         if (myAssemblyNames.Contains(assemblyName))
             return;
-         _loaded_assemblies.Add(assembly_name);
-         Assembly loaded_assembly = GetType().Assembly.FullName.Equals(assembly_name) ?
+         myAssemblyNames.Add(assemblyName);
+         Assembly loadedAssembly = GetType().Assembly.FullName.Equals(assemblyName) ?
             GetType().Assembly :
-            Assembly.LoadFrom(assembly_name);
+            Assembly.LoadFrom(assemblyName);
 
-         foreach (Type type in loaded_assembly.GetTypes())
-            if (_is_loadable_stream(type)) {
-               bool is_visible = true;
-               string dsl_name = type.Name; //createDefaultName(type.Name);
+         foreach (Type type in loadedAssembly.GetTypes())
+            if (!type.IsInterface
+               && !type.IsAbstract
+               && type.IsSubclassOf(typeof(ViewStream))
+               && type.IsPublic) {
+               bool isVisible = true;
+               string userRep = createDefaultName(type.Name);
                foreach (Attribute att in type.GetCustomAttributes(false))
                   if (att is DataStreamUserRespresentation) {
                      DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
-                     is_visible = dsatt.isVisible;
-                     dsl_name = is_visible ?
-                        dsatt.friendlyName == null ? dsl_name : dsatt.friendlyName
+                     isVisible = dsatt.isVisible;
+                     userRep = isVisible ?
+                        dsatt.friendlyName == null ? userRep : dsatt.friendlyName
                         : null;
                      break;
                   }
-               if (is_visible) {
-                  // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
-                  _dslopname_to_type_map[dsl_name] = type;
+               if (isVisible) {
+                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  myTypeLookupTable[userRep] = type;
                }
             }
       }
 
+      public static string createDefaultName(string typeName) {
+         if (!typeName.EndsWith("Stream"))
+            return typeName;
+         string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
+         string startString = new String(new char[] { retVal[0] }).ToLower();
+         retVal = retVal.Remove(0, 1);
+         retVal = retVal.Insert(0, startString);
+         return retVal;
+      }
+
+
 
       public static PrimitiveManager instance {
-         get { return _self = _self == null ? new PrimitiveManager() : _self; }
+         get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
       }
 
       public static void SetMarketDataClient(object value) {
@@ -96,20 +96,31 @@ namespace BCG {
          mdStreamFactory = new(value);
       }
 
-      public string dsl_to_type_name(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
-         return rv.Name;
+      /// <summary>
+      /// Returns the Type name corresponding to the primitive name
+      /// </summary>
+      /// <param name="friendlyName"></param>
+      /// <returns></returns>
+      public string find(string friendlyName) {
+         if (myTypeLookupTable[friendlyName] == null)
+            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
+         return (myTypeLookupTable[friendlyName] as Type).Name;
       }
 
+      /// <summary>
+      /// Constructs and returns a ViewStream of type corresponding to the friendly name
+      /// </summary>
+      /// <param name="friendlyName">friendly primitive name </param>
+      /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
+      /// is the desired one</param>
+      /// <returns></returns>
       public ViewStream build(string friendlyName, ICollection args) {
          object[] constructorArgs = null;
          if (args != null && args.Count > 0) {
             constructorArgs = new object[args.Count];
             args.CopyTo(constructorArgs, 0);
          }
-         var type = _dslopname_to_type_map[friendlyName] as Type
+         var type = myTypeLookupTable[friendlyName] as Type
             ?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
          if (type.HasCustomAttribute<LiveMarketDataStreamAttribute>()) {
             if (mdStreamFactory is null) {
@@ -124,14 +135,30 @@ namespace BCG {
             Activator.CreateInstance(type, constructorArgs) as ViewStream;
       }
 
-      public string[] get_dsl_op_names() {
-         return _dslopname_to_type_map.Keys.ToArray();
+      /// <summary>
+      /// Returns the name of all the primitives 
+      /// </summary>
+      /// <returns></returns>
+      public string[] getPrimitiveNames() {
+         string[] retVal = new string[myTypeLookupTable.Count];
+         myTypeLookupTable.Keys.CopyTo(retVal, 0);
+         return retVal;
       }
 
+      /// <summary>
+      /// Return the types of all the streams
+      /// </summary>
+      /// <returns></returns>
+      public Type[] getStreamTypes() {
+         Type[] retVal = new Type[myTypeLookupTable.Count];
+         myTypeLookupTable.Values.CopyTo(retVal, 0);
+         return retVal;
+      }
 
-      public ICollection get_dsl_op_cons_args(string dsl_op_name) {
+      public ICollection getConstructorSigns(string primitiveName) {
          ArrayList retVal = new ArrayList();
-         Type t = get_dsl_op_type(dsl_op_name);
+         Log.assert(myTypeLookupTable[primitiveName] != null);
+         Type t = myTypeLookupTable[primitiveName] as Type;
          foreach (ConstructorInfo ci in t.GetConstructors()) {
             string sign = "";
             sign = t.Name + " (";
@@ -145,18 +172,9 @@ namespace BCG {
          return retVal;
       }
 
-      public string get_dslop_assembly_name(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
-         return rv.Assembly.FullName;
+      public string getAssemblyName(string primitiveName) {
+         Log.assert(myTypeLookupTable[primitiveName] != null);
+         return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
       }
-
-      public string get_qualified_op_name(string dsl_op_name) {
-         Type dsl_op_type = get_dsl_op_type(dsl_op_name);
-         Log.assert(dsl_op_type != null);
-         return $"{dsl_op_type.Namespace}.{dsl_op_name}";
-      }
-
    }
 }

src/DataStreamSuite/DataStreamCore/IDataPacket.cs

@@ -13,6 +13,7 @@ namespace BCG {
       IBlueRowCollection newRows { get; }
       IBlueRowCollection addedRows { get; }
       IBlueRowCollection deletedRows { get; }
+      IReadOnlyDictionary<string, object> Metadata { get; }
       //this is an ICollection of IDictionary
       void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
    }

src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs

@@ -3,23 +3,23 @@ using System.IO;
 using System.Collections.Generic;
 using System.Linq;
 
-using BlueOrb.Data;
 using bts;
-using System.Threading;
 
-namespace BCG {
-   // This class is responsible for mantaining the state for a table. It gets the
-   // bytes from the pubsubserver via the handle_updates call 
-   // (called by VSD, - VSD is polling the SubscriberCOre for udpates), deserializes,
+namespace BCG
+{
+   // This class is responsible for mantaining the state fo a table. It gets the
+   // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
    // updates the state of the table, and fires the TableUpdateDelegate.
-   public class TableAccumulator2 : ITableAccumulator {
+   public class TableAccumulator2 : ITableAccumulator
+   {
       private readonly string _tag;
       private readonly ISubscriberCore _subscriber_core;
       IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
       private BlueTable _table;
       private TableUpdateDelegate _on_update_delegate;
 
-      public TableAccumulator2(string tag, ISubscriberCore sub_core) {
+      public TableAccumulator2(string tag, ISubscriberCore sub_core)
+      {
          _tag = tag;
          _subscriber_core = sub_core;
       }
@@ -28,54 +28,61 @@ namespace BCG {
 
       //public bool is_active() { return _on_update_delegate != null; }
 
-      public event TableUpdateDelegate on_update {
-         add {
+      public event TableUpdateDelegate on_update
+      {
+         add
+         {
             Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
-            if (_on_update_delegate == null) {
+            if (_on_update_delegate == null)
+            {
                //ie first subscription
                Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
                _subscriber_core.subscribe(_tag);
             }
             _on_update_delegate =
                Delegate.Combine(_on_update_delegate, value) as TableUpdateDelegate;
-            // a new subsriber should get a full callback immediatly (new rows, 0 deleted, all added)
             if (_table != null)
                value(_table, _table, new BlueTable(_table.schema));
          }
-         remove {
+         remove
+         {
             Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
-            if (_on_update_delegate != null) {
+            if (_on_update_delegate != null)
+            {
                _on_update_delegate =
                   Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
                int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
                Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
-               if (_on_update_delegate == null) {
+               if (_on_update_delegate == null)
+               {
                   _subscriber_core.unsubscribe(_tag);
-                  _table = null;
+                  // TODO
+                  //_table = null;
                }
             }
 
          }
       }
       private int _update_no = 0;
-      public void handle_updates(IEnumerable<byte[]> msgs) {
+      public void handle_updates(IEnumerable<byte[]> msgs)
+      {
          if (msgs == null || msgs.Count() == 0)
             return;
          Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
          IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
-         int i = 0;
-         foreach (var msg in msgs) {
-            bts.StreamIOMessage new_msg_txt = _deser.deserialize(msg);
+         foreach (var msg in msgs)
+         {
+            bts.StreamIOMessage str_msg = _deser.deserialize(msg);
             var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table);
-            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} object_hash: {GetHashCode()} thread_id: {Thread.CurrentThread.ManagedThreadId} msg: {i++ / msgs.Count()} msg_sz: {msg.Length} new_msg_txt: {new_msg_txt} pre_update_count: {state_txt}");
-
-            switch (new_msg_txt.op) {
+            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
+            switch (str_msg.op)
+            {
                case bts.StreamIOMessage.StreamOp.CREATE:
-                  _table = new BlueTable(new_msg_txt.create_message.rows);
+                  _table = new BlueTable(str_msg.create_message.rows);
                   break;
                case bts.StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
-                  _table.deleteRows(new_msg_txt.multiple_remove_add_message.removed_rows);
-                  _table.addRows(new_msg_txt.multiple_remove_add_message.added_rows);
+                  _table.deleteRows(str_msg.multiple_remove_add_message.removed_rows);
+                  _table.addRows(str_msg.multiple_remove_add_message.added_rows);
                   break;
             }
          }
@@ -84,16 +91,11 @@ namespace BCG {
          IBlueRowCollection d_rows, a_rows;
          //BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
          BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
-         Log.Gen.info($"TableAccumulator2.handle_updates firing update, _table: {_table.rows_id_string()} d_rows: {d_rows.rows_id_string()} a_rows: {a_rows.rows_id_string()}");
-         if (a_rows.Count == 0 && d_rows.Count == 0 && table_0 != null) {
-            Log.Gen.info($"TableAccmulator2.handle_updates ex! _table: {_table.ToDebugString()} table_0: {table_0.ToDebugString()} a_rows: {a_rows.ToDebugString()} d_rows: {d_rows.ToDebugString()}");
-            //same table was pubslished (maybe surpress it on the publisher or pubsubs)
-            return;
-         }
          _on_update_delegate?.Invoke(_table, a_rows, d_rows);
       }
 
-      public void update(Stream[] streams) {
+      public void update(Stream[] streams)
+      {
       }
 
       #endregion

src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs

@@ -30,7 +30,7 @@ namespace bts {
       }
    }
 
-   // This class is created by the SingleTagData for stream tag_type. Every DS tag has a PubsubServerSingleStreamTagHandler
+   // This class is created by the SingleTagData for stream tag_type. Every pub
 
    public class PubsubServerSingleStreamTagHandler : IPubSubServerTagHandler {
       // this is kept around to deserielize the incoming messages and keep the state of the table

src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs

@@ -20,7 +20,7 @@ public class PrimitiveExpression : StatementExpression {
    }
 
    public override string print(int startCol, IList<string> activeBindings) {
-      string prelude = "new " + PrimitiveManager.instance.dsl_to_type_name(myName) + " (";
+      string prelude = "new " + PrimitiveManager.instance.find(myName) + " (";
 
       return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
    }

src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs

@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
 
+using global::Spark.Core.Disposables;
+
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
 using System.Linq;

src/DataStreamSuite/DataStreamPublisher/WSPublishers/DatabentoPublisher.py

@@ -1,146 +0,0 @@
-"""
-Requires:
-   pip install -U databento
-   pip install websockets
-
-Sources:
-   https://databento.com/docs/quickstart/set-up
-   https://databento.com/docs/schemas-and-data-formats/bbo
-"""
-
-import datetime as dt
-import subprocess
-import sys
-import time
-
-try:
-   import databento as db
-except ImportError:
-   print("Installing missing dependency: databento")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
-   import databento as db
-
-from databento.common.error import BentoError
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-from databento_transformer import DatabentoTransformer
-from pubsub_client import PubSubClient
-
-
-API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
-DATASET = "GLBX.MDP3"
-SYMBOL = "ES.FUT"
-PUBSUB_WS_URL = "ws://localhost:19092/ws/"
-BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
-TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
-DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
-
-_pubsub_client = PubSubClient(PUBSUB_WS_URL)
-_transformer = DatabentoTransformer(
-   symbol=SYMBOL,
-   bbo_table_name=BBO_TABLE_NAME,
-   trades_table_name=TRADES_TABLE_NAME,
-   definitions_table_name=DEFINITIONS_TABLE_NAME,
-)
-
-
-def _publish_contract_definitions() -> None:
-   historical_client = db.Historical(API_KEY)
-   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
-   schema_range = dataset_range.get("schema", {}).get("definition", {})
-   available_end = schema_range.get("end", dataset_range["end"])
-
-   if isinstance(available_end, str):
-      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
-   else:
-      available_end_dt = available_end
-
-   # Databento's range end is exclusive. Back off slightly so the request stays
-   # strictly inside the available window.
-   end_dt = available_end_dt - dt.timedelta(seconds=1)
-   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
-   start = start_dt.isoformat()
-   end = end_dt.isoformat()
-
-   definition_data = historical_client.timeseries.get_range(
-      dataset=DATASET,
-      schema="definition",
-      stype_in="parent",
-      symbols=SYMBOL,
-      start=start,
-      end=end,
-   )
-
-   df = definition_data.to_df()
-   payload = _transformer.build_definitions_payload(df)
-   if payload is None:
-      print("No Databento contract definitions returned")
-      return
-
-   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
-   _pubsub_client.publish(payload)
-
-
-def on_record(record) -> None:
-   # Print every callback first so we can see whether Databento is flowing
-   # even if our row extraction logic misses a field shape.
-   print(record)
-
-   payloads = _transformer.transform_record(record)
-   if payloads:
-      for payload in payloads:
-         _pubsub_client.publish(payload)
-      return
-
-   print("Skipping record with no BBO/trade fields")
-
-
-def main() -> None:
-   _pubsub_client.start()
-   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
-   try:
-      _publish_contract_definitions()
-   except Exception as ex:
-      print(f"Contract definition publish failed: {ex}", flush=True)
-
-   while True:
-      try:
-         client = db.Live(key=API_KEY)
-
-         client.subscribe(
-            dataset=DATASET,
-            schema="bbo-1s",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-         client.subscribe(
-            dataset=DATASET,
-            schema="trades",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-
-         client.add_callback(on_record)
-         client.start()
-         print("Databento client started, waiting for BBO and trade records...", flush=True)
-         client.block_for_close()
-         print("Databento client closed cleanly", flush=True)
-         break
-      except BentoError as ex:
-         print(f"Databento live connection error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-      except Exception as ex:
-         print(f"Databento publisher unexpected error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-
-
-if __name__ == "__main__":
-   main()

src/DataStreamSuite/DataStreamPublisher/WSPublishers/databento_transformer.py

@@ -1,244 +0,0 @@
-import datetime as dt
-
-
-class DatabentoTransformer:
-   def __init__(
-      self,
-      symbol: str,
-      bbo_table_name: str,
-      trades_table_name: str,
-      definitions_table_name: str,
-   ):
-      self._symbol = symbol
-      self._bbo_table_name = bbo_table_name
-      self._trades_table_name = trades_table_name
-      self._definitions_table_name = definitions_table_name
-      self._instrument_id_to_symbol: dict[int, str] = {}
-      self._last_bbo_row: dict | None = None
-      self._trade_row_id = 0
-      self._trade_table_created = False
-
-   def transform_record(self, record) -> list[dict]:
-      payload = self._handle_bbo_record(record)
-      if payload is not None:
-         return [payload]
-
-      payload = self._handle_trade_record(record)
-      if payload is not None:
-         return [payload]
-
-      return []
-
-   def build_definitions_payload(self, definition_df) -> dict | None:
-      if definition_df is None or definition_df.empty:
-         return None
-
-      df = definition_df.reset_index()
-      if "instrument_class" in df.columns:
-         df = df[df["instrument_class"].astype(str) == "F"]
-      if "security_type" in df.columns:
-         df = df[df["security_type"].astype(str) == "FUT"]
-      if "raw_symbol" in df.columns:
-         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
-         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
-
-      rows: list[dict] = []
-      for idx, row in enumerate(df.itertuples(index=False), start=1):
-         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
-         security_name = str(getattr(row, "raw_symbol", ""))
-         if instrument_id > 0 and security_name:
-            self._instrument_id_to_symbol[instrument_id] = security_name
-
-         rows.append({
-            "id": idx,
-            "security_name": security_name,
-            "instrument_id": instrument_id,
-            "instrument_class": str(getattr(row, "instrument_class", "")),
-            "security_type": str(getattr(row, "security_type", "")),
-            "asset": str(getattr(row, "asset", "")),
-            "exchange": str(getattr(row, "exchange", "")),
-            "currency": str(getattr(row, "currency", "")),
-            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
-            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
-            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
-            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
-            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
-            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
-            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
-            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
-            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
-            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
-            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
-            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
-            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
-            "underlying": str(getattr(row, "underlying", "")),
-         })
-
-      if not rows:
-         return None
-
-      return {
-         "op": "create_table",
-         "table_name": self._definitions_table_name,
-         "rows": rows,
-      }
-
-   def _handle_bbo_record(self, record) -> dict | None:
-      row = self._to_bbo_row(record)
-      if row is None:
-         return None
-
-      if self._last_bbo_row is None:
-         payload = {
-            "op": "create_table",
-            "table_name": self._bbo_table_name,
-            "rows": [row],
-         }
-      else:
-         payload = {
-            "op": "multiple_remove_add",
-            "table_name": self._bbo_table_name,
-            "removed_rows": [self._last_bbo_row],
-            "added_rows": [row],
-         }
-
-      self._last_bbo_row = row
-      return payload
-
-   def _handle_trade_record(self, record) -> dict | None:
-      row = self._to_trade_row(record)
-      if row is None:
-         return None
-
-      row["id"] = self._trade_row_id
-      self._trade_row_id += 1
-
-      if not self._trade_table_created:
-         payload = {
-            "op": "create_table",
-            "table_name": self._trades_table_name,
-            "rows": [row],
-         }
-         self._trade_table_created = True
-         return payload
-
-      return {
-         "op": "multiple_remove_add",
-         "table_name": self._trades_table_name,
-         "removed_rows": [],
-         "added_rows": [row],
-      }
-
-   def _to_bbo_row(self, record) -> dict | None:
-      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
-      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
-      if bid_px is None and ask_px is None:
-         return None
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "bid_px": float(bid_px or 0.0),
-         "ask_px": float(ask_px or 0.0),
-         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
-         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
-      }
-
-   def _to_trade_row(self, record) -> dict | None:
-      price = self._get_attr(record, "price", "px")
-      size = self._get_attr(record, "size")
-      if price is None and size is None:
-         return None
-
-      side = self._get_attr(record, "side", "aggressor_side", default="")
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "price": float(price or 0.0),
-         "size": int(size or 0),
-         "side": str(side),
-      }
-
-   def _get_security_name(self, record) -> str:
-      instrument_id = self._get_attr(record, "instrument_id", default=0)
-      try:
-         instrument_id_int = int(instrument_id or 0)
-      except (TypeError, ValueError):
-         instrument_id_int = 0
-
-      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
-         return self._instrument_id_to_symbol[instrument_id_int]
-
-      value = self._get_attr(
-         record,
-         "symbol",
-         "raw_symbol",
-         "stype_out_symbol",
-         "stype_in_symbol",
-         default=self._symbol,
-      )
-      return str(value)
-
-   def _get_timestamp(self, record) -> int:
-      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
-      try:
-         return int(value)
-      except (TypeError, ValueError):
-         return 0
-
-   def _get_attr(self, record, *names, default=None):
-      for name in names:
-         if hasattr(record, name):
-            value = getattr(record, name)
-            if value is not None:
-               return value
-      return default
-
-   def _get_level_attr(self, record, name, default=None):
-      levels = self._get_attr(record, "levels", default=None)
-      if levels:
-         level0 = levels[0]
-         if hasattr(level0, name):
-            value = getattr(level0, name)
-            if value is not None:
-               return value
-      return default
-
-   def _ns_to_iso(self, timestamp_ns) -> str:
-      if timestamp_ns is None:
-         return ""
-
-      if hasattr(timestamp_ns, "isoformat"):
-         try:
-            if hasattr(timestamp_ns, "to_pydatetime"):
-               timestamp_dt = timestamp_ns.to_pydatetime()
-            else:
-               timestamp_dt = timestamp_ns
-
-            if timestamp_dt is None:
-               return ""
-
-            if getattr(timestamp_dt, "tzinfo", None) is None:
-               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
-            return timestamp_dt.isoformat()
-         except (TypeError, ValueError, AttributeError):
-            pass
-
-      try:
-         ts_int = int(timestamp_ns)
-         if ts_int <= 0:
-            return ""
-         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
-      except (TypeError, ValueError, OSError, OverflowError):
-         return ""
-
-   def _nano_to_float(self, value) -> float:
-      try:
-         return float(value) / 1_000_000_000.0
-      except (TypeError, ValueError):
-         return 0.0

src/DataStreamSuite/DataStreamPublisher/WSPublishers/pubsub_client.py

@@ -1,58 +0,0 @@
-import asyncio
-import json
-import queue
-import threading
-
-import websockets
-
-
-class PubSubClient:
-   def __init__(self, ws_url: str):
-      self._ws_url = ws_url
-      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
-      self._sender_thread: threading.Thread | None = None
-
-   def start(self) -> threading.Thread:
-      if self._sender_thread is None or not self._sender_thread.is_alive():
-         self._sender_thread = self._start_sender_thread()
-      return self._sender_thread
-
-   def publish(self, payload: dict) -> None:
-      self._outgoing_queue.put(payload)
-
-   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
-      self.publish({
-         "op": "create_table",
-         "table_name": table_name,
-         "rows": rows,
-      })
-
-   def publish_multiple_remove_add(
-      self,
-      table_name: str,
-      removed_rows: list[dict],
-      added_rows: list[dict],
-   ) -> None:
-      self.publish({
-         "op": "multiple_remove_add",
-         "table_name": table_name,
-         "removed_rows": removed_rows,
-         "added_rows": added_rows,
-      })
-
-   def _start_sender_thread(self) -> threading.Thread:
-      thread = threading.Thread(target=self._run_send_loop, daemon=True)
-      thread.start()
-      return thread
-
-   def _run_send_loop(self) -> None:
-      asyncio.run(self._ws_send_loop())
-
-   async def _ws_send_loop(self) -> None:
-      async with websockets.connect(self._ws_url) as ws:
-         print(f"Connected to PubSubServer websocket: {self._ws_url}")
-         while True:
-            payload = await asyncio.to_thread(self._outgoing_queue.get)
-            body = json.dumps(payload)
-            print(body)
-            await ws.send(body)

src/DataStreamSuite/DataStreamPublisher/appsettings.json

@@ -59,9 +59,14 @@
           "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
         },
         {
           "TagType": ".*",
@@ -85,4 +90,4 @@
       }
     ]
   }
-}
+}

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -21,55 +21,11 @@ using System.Reactive.Subjects;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Windows.Forms;
-using BlueOrb.Pubsub.Diagnostics;
 
 namespace BCG
 {
    public partial class MainForm : System.Windows.Forms.Form
    {
-      private const string LastQueryRegistryKey = "LastQuery";
-      private const string RecentSavedFilesRegistryKey = "RecentSavedFiles";
-      private const string DateTimeDisplayFormat = "M/d/yyyy h:mm:ss.fff tt";
-      private const int MaxRecentSavedFiles = 10;
-      private static readonly Color QueryEditorDefaultColor = Color.White;
-      private static readonly Color QueryEditorCommentColor = Color.FromArgb(106, 115, 125);
-      private static readonly Color QueryEditorOperatorColor = Color.FromArgb(86, 156, 214);
-      private static readonly Color QueryEditorAssignmentNameColor = Color.FromArgb(78, 201, 176);
-      private static readonly Color GridBackgroundColor = Color.FromArgb(37, 37, 38);
-      private static readonly Color GridAltRowColor = Color.FromArgb(43, 43, 48);
-      private static readonly Color GridHeaderColor = Color.FromArgb(62, 62, 66);
-      private static readonly Color GridGridLineColor = Color.FromArgb(58, 58, 61);
-      private static readonly Color GridSelectionColor = Color.FromArgb(9, 71, 113);
-      private static readonly Color GridSelectionForeColor = Color.White;
-      private static readonly Color GridTimestampColor = Color.FromArgb(206, 215, 222);
-      private static readonly Color GridIdentifierColor = Color.FromArgb(156, 163, 175);
-      private static readonly Color GridIndicatorColor = Color.FromArgb(156, 220, 254);
-
-      class PrimitiveTreeNodeComparer : IComparer
-      {
-         public int Compare(object? x, object? y)
-         {
-            var left = x as TreeNode;
-            var right = y as TreeNode;
-            if (left is null || right is null)
-               return 0;
-
-            string left_key = _sort_key(left);
-            string right_key = _sort_key(right);
-            return StringComparer.OrdinalIgnoreCase.Compare(left_key, right_key);
-         }
-
-         static string _sort_key(TreeNode node)
-         {
-            if (node.Parent is null)
-               return node.Text;
-
-            if (node.Tag is string primitive_name)
-               return primitive_name;
-
-            return node.Text;
-         }
-      }
       // static async Task _run() {
       //    var login = "bts";
       //    var passwd = String.Empty;
@@ -84,8 +40,6 @@ namespace BCG
       private string? myLastFileName;
       private ViewStream? myCurrentStream;
       private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
-      private readonly List<string> _recent_saved_files = new List<string>();
-      private ToolStripSeparator? _recent_saved_files_separator;
       // private string mySite = SitesCore.getMySiteName ();
       private string mySite = "TODO";
       private string myLastClassName = "MyView";
@@ -95,20 +49,11 @@ namespace BCG
       private TextWriter? myConsoleOutput;
       IDataPacket? pendingData;
       IDisposable? titleSub;
-      readonly ToolTip _primitive_tool_tip = new ToolTip();
-      readonly Dictionary<string, string?> _primitive_source_path_cache = new Dictionary<string, string?>();
-      Dictionary<string, string?>? _source_file_index;
-      string? _repo_root_path;
-      char? _last_primitive_search_char;
-      TreeNode? _last_primitive_search_assembly_node;
-      bool _is_applying_query_syntax_highlighting;
-      HashSet<string>? _dsl_operator_names;
 
       public MainForm()
       {
          //Task.Run(async () => await _run());
          InitializeComponent();
-         _apply_grid_theme();
          // myKQ = ConfigurationRequester.getApplicationQualifier();         
          myKQ = "TODO";
          var handleCreatedSubject = new Subject<bool>();
@@ -127,12 +72,13 @@ namespace BCG
          myWindowState = new PersistWindowState();
          myWindowState.parent = this;
          myWindowState.saveState = false;
-         myWindowState.LoadState += myLoadPersistedState;
-         myWindowState.SaveState += mySavePersistedState;
 
          this.Load += new EventHandler(myLoad);
          this.Closing += new CancelEventHandler(myClosing);
-         myQueryTextBox.TextChanged += (o, e) => _apply_query_syntax_highlighting();
+         myQueryTextBox.TextChanged += (o, e) =>
+         {
+            myQueryTextBox.ForeColor = Color.White;
+         };
       }
 
       event Action? titleInvalidated;
@@ -140,20 +86,14 @@ namespace BCG
       public string query_text
       {
          get { return myQueryTextBox.Text; }
-         set
-         {
-            myQueryTextBox.Text = value;
-            _apply_query_syntax_highlighting();
-         }
+         set { myQueryTextBox.Text = value; }
       }
 
       /// <summary>
       /// Clean up any resources being used.
       /// </summary>
-      protected override void Dispose(bool disposing)
-      {
-         if (disposing)
-         {
+      protected override void Dispose(bool disposing) {
+         if (disposing) {
             Disposable.Dispose(ref titleSub);
             components?.Dispose();
          }
@@ -164,7 +104,7 @@ namespace BCG
       {
          // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
          //ViewStream vs = new PubSubServerStateStream();
-         ViewStream vs = new PubSubServerStateStream(ViewStreamDatabase.instance);
+         ViewStream vs = new PubSubServerStateStream(new SubscriberCoreFactory(), ViewStreamDatabase.instance);
          return new FilterStream(vs, "tag~='DS_'");
          //return new LiteralStream("tag/String", "DS_table1;DS_table2");
       }
@@ -174,8 +114,7 @@ namespace BCG
          titleInvalidated?.Invoke();
       }
 
-      void refreshTitle(IBlueRowCollection rows)
-      {
+      void refreshTitle(IBlueRowCollection rows) {
          string route_txt = String.Empty;
          foreach (IBlueRow r in rows)
          {
@@ -196,11 +135,11 @@ namespace BCG
       private IDictionary myGetLibraryData()
       {
          SortedList retVal = new SortedList();
-         string[] primitiveNames = PrimitiveManager.instance.get_dsl_op_names();
+         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
          foreach (string pName in primitiveNames)
          {
-            string aName = PrimitiveManager.instance.get_dslop_assembly_name(pName);
-            ICollection cNames = PrimitiveManager.instance.get_dsl_op_cons_args(pName);
+            string aName = PrimitiveManager.instance.getAssemblyName(pName);
+            ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
             if (!retVal.Contains(aName))
                retVal.Add(aName, new SortedList());
             (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
@@ -212,65 +151,6 @@ namespace BCG
          return retVal;
       }
 
-      HashSet<string> _get_dsl_operator_names()
-      {
-         if (_dsl_operator_names != null)
-            return _dsl_operator_names;
-
-         _dsl_operator_names = new HashSet<string>(
-            PrimitiveManager.instance.get_dsl_op_names().Cast<string>(),
-            StringComparer.OrdinalIgnoreCase);
-         return _dsl_operator_names;
-      }
-
-      void _apply_query_syntax_highlighting()
-      {
-         if (_is_applying_query_syntax_highlighting || myQueryTextBox.IsDisposed)
-            return;
-
-         _is_applying_query_syntax_highlighting = true;
-         try
-         {
-            int selection_start = myQueryTextBox.SelectionStart;
-            int selection_length = myQueryTextBox.SelectionLength;
-
-            myQueryTextBox.SuspendLayout();
-            myQueryTextBox.SelectAll();
-            myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"^\s*([A-Za-z_]\w*)\s*=", RegexOptions.Multiline))
-            {
-               Group group = match.Groups[1];
-               myQueryTextBox.Select(group.Index, group.Length);
-               myQueryTextBox.SelectionColor = QueryEditorAssignmentNameColor;
-            }
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"\b([A-Za-z_]\w*)\s*\("))
-            {
-               Group group = match.Groups[1];
-               if (!_get_dsl_operator_names().Contains(group.Value))
-                  continue;
-               myQueryTextBox.Select(group.Index, group.Length);
-               myQueryTextBox.SelectionColor = QueryEditorOperatorColor;
-            }
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"//.*$", RegexOptions.Multiline))
-            {
-               myQueryTextBox.Select(match.Index, match.Length);
-               myQueryTextBox.SelectionColor = QueryEditorCommentColor;
-            }
-
-            myQueryTextBox.Select(selection_start, selection_length);
-            if (selection_length == 0)
-               myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
-            myQueryTextBox.ResumeLayout();
-         }
-         finally
-         {
-            _is_applying_query_syntax_highlighting = false;
-         }
-      }
-
 
       private void myLoadAssemblies()
       {
@@ -281,18 +161,13 @@ namespace BCG
             TreeNode assemblyNode = new TreeNode(de.Key as String);
             foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
             {
-               string primitive_name = innerDe.Key as string;
-               TreeNode primitiveNode = new TreeNode(primitive_name)
-               {
-                  Tag = primitive_name
-               };
+               TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
                foreach (string con in (innerDe.Value as ICollection))
                   primitiveNode.Nodes.Add(con);
                assemblyNode.Nodes.Add(primitiveNode);
             }
             myTreeView.Nodes.Add(assemblyNode);
          }
-         myTreeView.Sort();
          /*
          myPrimitivesTextBox.Clear ();
          ArrayList al = new ArrayList(PrimitiveManager.instance.getPrimitiveNames());
@@ -307,8 +182,7 @@ namespace BCG
 
       private void myHandleTableListChange(object o)
       {
-         BeginInvoke(() =>
-         {
+         BeginInvoke(() => {
             tagRows = ((IDataPacket)o).newRows;
             myUpdateTableList();
          });
@@ -321,19 +195,6 @@ namespace BCG
          get { return myQueryTextBox.Height; }
          set { myQueryTextBox.Height = value; }
       }
-
-      private void myLoadPersistedState(object sender, BlueRegistry blueReg)
-      {
-         myQueryTextBox.Text = blueReg.getValue(RegistryRoot.HKCU, LastQueryRegistryKey, string.Empty)?.ToString() ?? string.Empty;
-         _load_recent_saved_files(blueReg.getValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Empty)?.ToString());
-         _refresh_recent_saved_files_menu();
-      }
-
-      private void mySavePersistedState(object sender, BlueRegistry blueReg)
-      {
-         blueReg.setValue(RegistryRoot.HKCU, LastQueryRegistryKey, myQueryTextBox.Text ?? string.Empty);
-         blueReg.setValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Join("|", _recent_saved_files));
-      }
       #endregion
 
       private Stopwatch _stopwatch = new Stopwatch();
@@ -426,7 +287,6 @@ namespace BCG
             }
             myLastFileName = fileName;
             this.Text = fileName;
-            _track_recent_saved_file(fileName);
          }
          catch (Exception exc)
          {
@@ -511,122 +371,6 @@ namespace BCG
          }
       }
 
-      private void myApplyDateTimeFormatting(IBlueTableSchema schema)
-      {
-         var dateTimeColumns = new HashSet<string>(
-            schema.columnInfo
-               .Where(ci => ci.colType == typeof(DateTime))
-               .Select(ci => ci.colName));
-
-         foreach (DataGridViewColumn column in myDataGrid.Columns)
-         {
-            if (!dateTimeColumns.Contains(column.DataPropertyName) && !dateTimeColumns.Contains(column.Name))
-               continue;
-
-            column.DefaultCellStyle.Format = DateTimeDisplayFormat;
-            column.Width = Math.Max(column.Width, 220);
-         }
-      }
-
-      void _apply_grid_theme()
-      {
-         myDataGrid.BackgroundColor = GridBackgroundColor;
-         myDataGrid.BorderStyle = BorderStyle.None;
-         myDataGrid.EnableHeadersVisualStyles = false;
-         myDataGrid.GridColor = GridGridLineColor;
-         myDataGrid.CellBorderStyle = DataGridViewCellBorderStyle.SingleHorizontal;
-         myDataGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect;
-         myDataGrid.MultiSelect = false;
-         myDataGrid.RowHeadersVisible = false;
-         myDataGrid.ColumnHeadersBorderStyle = DataGridViewHeaderBorderStyle.Single;
-         myDataGrid.RowHeadersBorderStyle = DataGridViewHeaderBorderStyle.None;
-         myDataGrid.ColumnHeadersHeight = 36;
-         myDataGrid.RowTemplate.Height = 24;
-         myDataGrid.Font = new Font("Consolas", 9F, FontStyle.Regular, GraphicsUnit.Point);
-
-         myDataGrid.DefaultCellStyle.BackColor = GridBackgroundColor;
-         myDataGrid.DefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.DefaultCellStyle.SelectionBackColor = GridSelectionColor;
-         myDataGrid.DefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
-         myDataGrid.DefaultCellStyle.Padding = new Padding(4, 1, 4, 1);
-
-         myDataGrid.AlternatingRowsDefaultCellStyle.BackColor = GridAltRowColor;
-         myDataGrid.AlternatingRowsDefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionBackColor = GridSelectionColor;
-         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
-
-         myDataGrid.ColumnHeadersDefaultCellStyle.BackColor = GridHeaderColor;
-         myDataGrid.ColumnHeadersDefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionBackColor = GridHeaderColor;
-         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionForeColor = Color.White;
-         myDataGrid.ColumnHeadersDefaultCellStyle.Font = new Font("Consolas", 9F, FontStyle.Bold, GraphicsUnit.Point);
-         myDataGrid.ColumnHeadersDefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
-         myDataGrid.ColumnHeadersDefaultCellStyle.Padding = new Padding(6, 2, 6, 2);
-      }
-
-      void _apply_grid_column_styling(IBlueTableSchema schema)
-      {
-         var column_types = schema.columnInfo.ToDictionary(ci => ci.colName, ci => ci.colType, StringComparer.OrdinalIgnoreCase);
-         foreach (DataGridViewColumn column in myDataGrid.Columns)
-         {
-            if (!column_types.TryGetValue(column.DataPropertyName, out var col_type) &&
-                !column_types.TryGetValue(column.Name, out col_type))
-            {
-               continue;
-            }
-
-            column.SortMode = DataGridViewColumnSortMode.NotSortable;
-
-            if (_is_numeric_type(col_type))
-            {
-               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleRight;
-            }
-            else
-            {
-               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
-            }
-
-            if (col_type == typeof(DateTime))
-            {
-               column.DefaultCellStyle.ForeColor = GridTimestampColor;
-            }
-            else if (string.Equals(column.HeaderText, "id", StringComparison.OrdinalIgnoreCase))
-            {
-               column.DefaultCellStyle.ForeColor = GridIdentifierColor;
-            }
-            else if (_looks_like_indicator(column.HeaderText))
-            {
-               column.DefaultCellStyle.ForeColor = GridIndicatorColor;
-            }
-            else
-            {
-               column.DefaultCellStyle.ForeColor = Color.White;
-            }
-         }
-
-         static bool _is_numeric_type(Type type)
-         {
-            return type == typeof(int) ||
-                   type == typeof(long) ||
-                   type == typeof(float) ||
-                   type == typeof(double) ||
-                   type == typeof(decimal) ||
-                   type == typeof(short) ||
-                   type == typeof(uint) ||
-                   type == typeof(ulong);
-         }
-
-         static bool _looks_like_indicator(string column_name)
-         {
-            return column_name.Contains("rsi", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("macd", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("atr", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("moving_average", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("signal", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("histogram", StringComparison.OrdinalIgnoreCase);
-         }
-      }
-
       // this is the stream callback
       private void myCallback(object o)
       {
@@ -642,8 +386,7 @@ namespace BCG
       private void myTimer_Tick(object sender, System.EventArgs e)
       {
          var dp = Interlocked.Exchange(ref pendingData, null);
-         if (dp is null)
-         {
+         if (dp is null) {
             return;
          }
          _stopwatch.Stop();
@@ -660,10 +403,6 @@ namespace BCG
          int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
 
          myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
-         _apply_grid_theme();
-         myApplyDateTimeFormatting(dp.newRows.schema);
-         _apply_grid_column_styling(dp.newRows.schema);
-         myDataGrid.ClearSelection();
          var force = false;
          if (myChangeStyle && force)
          {
@@ -736,7 +475,7 @@ namespace BCG
       private void myMenuItemOpenClick(object sender, System.EventArgs e)
       {
          OpenFileDialog od = new OpenFileDialog();
-         od.Filter = "All files (*.*)|*.*|C# class (*.cs)|*.cs";
+         od.Filter = "C# class (*.cs)|*.cs";
 
          if (od.ShowDialog() == DialogResult.OK)
             loadFile(od.FileName);
@@ -748,7 +487,6 @@ namespace BCG
             w.Write(myCSharpText.Text);
          myLastFileName = fileName;
          this.Text = fileName;
-         _track_recent_saved_file(fileName);
       }
 
       private void myMenuItemSaveClick(object sender, System.EventArgs e)
@@ -771,75 +509,6 @@ namespace BCG
             mySave(sd.FileName);
       }
 
-      private void _load_recent_saved_files(string? serialized_paths)
-      {
-         _recent_saved_files.Clear();
-         if (string.IsNullOrWhiteSpace(serialized_paths))
-            return;
-
-         foreach (string path in serialized_paths.Split('|'))
-         {
-            if (string.IsNullOrWhiteSpace(path))
-               continue;
-            if (_recent_saved_files.Contains(path, StringComparer.OrdinalIgnoreCase))
-               continue;
-            _recent_saved_files.Add(path);
-            if (_recent_saved_files.Count >= MaxRecentSavedFiles)
-               break;
-         }
-      }
-
-      private void _track_recent_saved_file(string file_name)
-      {
-         _recent_saved_files.RemoveAll(path => string.Equals(path, file_name, StringComparison.OrdinalIgnoreCase));
-         _recent_saved_files.Insert(0, file_name);
-         if (_recent_saved_files.Count > MaxRecentSavedFiles)
-            _recent_saved_files.RemoveRange(MaxRecentSavedFiles, _recent_saved_files.Count - MaxRecentSavedFiles);
-         _refresh_recent_saved_files_menu();
-      }
-
-      private void _refresh_recent_saved_files_menu()
-      {
-         var file_menu = myToolStripMenuItemFile.DropDownItems;
-         for (int i = file_menu.Count - 1; i >= 0; i--)
-         {
-            if (file_menu[i] is ToolStripMenuItem menu_item && Equals(menu_item.Tag, "recent-saved-file"))
-               file_menu.RemoveAt(i);
-         }
-
-         if (_recent_saved_files_separator != null)
-         {
-            file_menu.Remove(_recent_saved_files_separator);
-            _recent_saved_files_separator.Dispose();
-            _recent_saved_files_separator = null;
-         }
-
-         if (_recent_saved_files.Count == 0)
-            return;
-
-         int insert_index = file_menu.IndexOf(myToolStripMenuItemExit);
-         if (insert_index < 0)
-            insert_index = file_menu.Count;
-
-         _recent_saved_files_separator = new ToolStripSeparator();
-         file_menu.Insert(insert_index, _recent_saved_files_separator);
-         insert_index++;
-
-         foreach (string path in _recent_saved_files)
-         {
-            var menu_item = new ToolStripMenuItem(Path.GetFileName(path))
-            {
-               Tag = "recent-saved-file",
-               ToolTipText = path,
-               BackColor = Color.FromArgb(36, 36, 49),
-               ForeColor = Color.Lime
-            };
-            menu_item.Click += (_, __) => loadFile(path);
-            file_menu.Insert(insert_index, menu_item);
-            insert_index++;
-         }
-      }
-
       private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
       {
          if (myTableListBox.SelectedItem != null)
@@ -930,7 +599,6 @@ namespace BCG
          }
          Console.WriteLine("Redirecting console ouput to this window...");
 
-         myTreeView.TreeViewNodeSorter = new PrimitiveTreeNodeComparer();
          myLoadAssemblies();
 
          // ViewStreamDatabase.instance = new ViewStreamDatabase();
@@ -938,9 +606,6 @@ namespace BCG
          myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
          myTabControl.SelectedIndex = 3;
          myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
-         myTreeView.KeyPress += myHandleTreeViewKeyPress;
-         myTreeView.NodeMouseHover += myHandleTreeNodeMouseHover;
-         myTreeView.MouseLeave += myHandleTreeViewMouseLeave;
          //do this lazily so we don't have to have bps, etc up to run dsqa
          //_library = new LibraryForm(this);
       }
@@ -981,138 +646,12 @@ namespace BCG
          {
             if (myTreeView.SelectedNode.LastNode == null)
             {
-               string primitive_name = myTreeView.SelectedNode.Parent.Tag as string ?? myTreeView.SelectedNode.Parent.Text;
-               string sn = primitive_name + "()";
+               string sn = myTreeView.SelectedNode.Parent.Text + "()";
                myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
             }
          }
       }
 
-      private void myHandleTreeViewKeyPress(object? sender, KeyPressEventArgs e)
-      {
-         if (!Char.IsLetterOrDigit(e.KeyChar))
-            return;
-
-         char search_char = Char.ToLowerInvariant(e.KeyChar);
-         TreeNode? assembly_node = _get_selected_assembly_node();
-         if (assembly_node is null)
-            return;
-
-         int start_idx = 0;
-         if (_last_primitive_search_char == search_char &&
-             ReferenceEquals(_last_primitive_search_assembly_node, assembly_node) &&
-             myTreeView.SelectedNode?.Parent == assembly_node)
-         {
-            start_idx = myTreeView.SelectedNode.Index + 1;
-         }
-
-         TreeNode? match_node = _find_matching_primitive_node(assembly_node, search_char, start_idx) ??
-                                _find_matching_primitive_node(assembly_node, search_char, 0);
-
-         if (match_node is not null)
-         {
-            myTreeView.SelectedNode = match_node;
-            match_node.EnsureVisible();
-            _last_primitive_search_char = search_char;
-            _last_primitive_search_assembly_node = assembly_node;
-            e.Handled = true;
-         }
-      }
-
-      private TreeNode? _get_selected_assembly_node()
-      {
-         if (myTreeView.SelectedNode is null)
-            return myTreeView.Nodes.Count > 0 ? myTreeView.Nodes[0] : null;
-
-         return myTreeView.SelectedNode.Parent ?? myTreeView.SelectedNode;
-      }
-
-      private TreeNode? _find_matching_primitive_node(TreeNode assembly_node, char search_char, int start_idx)
-      {
-         for (int i = start_idx; i < assembly_node.Nodes.Count; i++)
-         {
-            TreeNode primitive_node = assembly_node.Nodes[i];
-            string primitive_name = primitive_node.Tag as string ?? primitive_node.Text;
-            if (primitive_name.Length > 0 && Char.ToLowerInvariant(primitive_name[0]) == search_char)
-               return primitive_node;
-         }
-         return null;
-      }
-
-      private void myHandleTreeNodeMouseHover(object? sender, TreeNodeMouseHoverEventArgs e)
-      {
-         if (e.Node?.Tag is not string primitive_name)
-         {
-            _primitive_tool_tip.Hide(myTreeView);
-            return;
-         }
-
-         Type? primitive_type = PrimitiveManager.instance.get_dsl_op_type(primitive_name);
-         if (primitive_type is null)
-         {
-            _primitive_tool_tip.Hide(myTreeView);
-            return;
-         }
-
-         string tooltip_text = $"Namespace: {primitive_type.Namespace ?? "(none)"}";
-         string? source_path = _get_source_file_path(primitive_type);
-         if (!String.IsNullOrWhiteSpace(source_path))
-            tooltip_text += $"\nSource: {source_path}";
-
-         Point mouse_point = myTreeView.PointToClient(Cursor.Position);
-         _primitive_tool_tip.Show(tooltip_text, myTreeView, mouse_point.X + 12, mouse_point.Y + 12, 8000);
-      }
-
-      private void myHandleTreeViewMouseLeave(object? sender, EventArgs e)
-      {
-         _primitive_tool_tip.Hide(myTreeView);
-      }
-
-      private string? _get_source_file_path(Type primitive_type)
-      {
-         string cache_key = primitive_type.FullName ?? primitive_type.Name;
-         if (_primitive_source_path_cache.TryGetValue(cache_key, out string? cached_path))
-            return cached_path;
-
-         Dictionary<string, string?> source_file_index = _source_file_index ??= _build_source_file_index();
-         if (!source_file_index.TryGetValue(primitive_type.Name, out string? source_path))
-            return null;
-         _primitive_source_path_cache[cache_key] = source_path;
-         return source_path;
-      }
-
-      private Dictionary<string, string?> _build_source_file_index()
-      {
-         Dictionary<string, string?> source_file_index = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
-         string? repo_root = _repo_root_path ??= _find_repo_root();
-         if (String.IsNullOrWhiteSpace(repo_root))
-            return source_file_index;
-
-         string source_root = Path.Combine(repo_root, "src", "DataStreamSuite");
-         if (!Directory.Exists(source_root))
-            return source_file_index;
-
-         foreach (string file_path in Directory.EnumerateFiles(source_root, "*.cs", SearchOption.AllDirectories))
-         {
-            string type_name = Path.GetFileNameWithoutExtension(file_path);
-            if (!source_file_index.ContainsKey(type_name))
-               source_file_index[type_name] = file_path;
-         }
-         return source_file_index;
-      }
-
-      private string? _find_repo_root()
-      {
-         string? curr_path = AppDomain.CurrentDomain.BaseDirectory;
-         while (!String.IsNullOrWhiteSpace(curr_path))
-         {
-            if (Directory.Exists(Path.Combine(curr_path, "src", "DataStreamSuite")))
-               return curr_path;
-            curr_path = Directory.GetParent(curr_path)?.FullName;
-         }
-         return null;
-      }
-
       private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
       {
          Close();
@@ -1142,7 +681,7 @@ namespace BCG
          DialogResult dr = ofd.ShowDialog(this);
          if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
          {
-            PrimitiveManager.instance.load_streams_from_assembly(ofd.FileName);
+            PrimitiveManager.instance.loadAssembly(ofd.FileName);
             myLoadAssemblies();
          }
       }
@@ -1229,8 +768,7 @@ namespace BCG
          s = new MutableAppendStream(s, s);
          s.onChange += OnChange;
 
-         void OnChange(object? obj)
-         {
+         void OnChange(object? obj) {
             s.onChange -= OnChange;
             //s = new MutableAppendStream(s, s);
             var rowCollection = ((IDataPacket)obj!).newRows;

src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs

@@ -8,6 +8,7 @@ using Bts.PubSub.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 
+using Spark.Api.MarketData;
 using Spark.Api.Services;
 using Spark.Core.Services;
 
@@ -25,6 +26,10 @@ static class Program {
          host = HostBuilderExtensions.CreateBtsBuilder(args)
             .ConfigureServices(ConfigureServices)
             .Build();
+         var mdClient = host.Services.GetService<IMarketDataClient>();
+         if (mdClient is not null) {
+            PrimitiveManager.SetMarketDataClient(mdClient);
+         }
          await host.StartAsync();
          var winFormsThread = new Thread(RunWinFormsApp);
          winFormsThread.SetApartmentState(ApartmentState.STA);

src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json

@@ -5,7 +5,8 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
-  "Spark": {},
+  "Spark": {
+  },
   "SparkApi": {
     "IsEnabled": false,
     "Credentials": {
@@ -52,16 +53,21 @@
           "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
         },
         {
           "TagType": ".*",
           "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
+         "Class": "bts.PubsubServerGenericDataHandler"
         }
       ]
     }
   }
-}
+}

src/DataStreamSuite/DataStreamSuite.sln

@@ -13,8 +13,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "PubSub\PubSub.csp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSubServer", "PubSubServer\PubSubServer.csproj", "{0DBD814A-E17F-4836-8EEE-6E91B00C578F}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PersistenceServer", "PersistenceServer\PersistenceServer.csproj", "{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "BooleanExpressions\BooleanExpressions.csproj", "{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{88E70136-91AE-4F56-B243-8F4642B46D6A}"
@@ -45,10 +43,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DSPublisher", "DSPublisher\DSPublisher.csproj", "{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIBridgeServer", "APIBridgeServer\APIBridgeServer.csproj", "{3B14C417-531A-4E39-80C1-A4C14BD000D5}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -119,18 +113,6 @@ Global
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x64.Build.0 = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.ActiveCfg = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.Build.0 = Release|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -311,30 +293,6 @@ Global
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/DataStreamSuite/DataStreamTests/BarBuilderStreamTests.cs

@@ -0,0 +1,218 @@
+// namespace Bts.DataStreamTests;
+
+// using BCG;
+
+// using BlueOrb.Operators.Core;
+
+// [TestFixture]
+// public class BarBuilderStreamTests {
+//    static readonly DateTime T0 = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+
+//    [Test]
+//    public void TimeBars_FromTrades_AcrossBoundary() {
+//       var schema = TradeSchema();
+//       var driver = new TestDriverStream(schema);
+//       var bars = new BarBuilderStream(driver, "TIME", 60, "trade_price", "trade_size");
+//       var rows = CaptureAddedRows(bars);
+
+//       var sent = new List<IBlueRow>();
+//       sent.Add(TradeRow(schema, 1, T0.AddSeconds(10), "BTCUSD", 100, 2));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 2, T0.AddSeconds(20), "BTCUSD", 101, 3));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 3, T0.AddMinutes(1).AddSeconds(5), "BTCUSD", 99, 1));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 4, T0.AddMinutes(2), "BTCUSD", 102, 1));
+//       driver.Reset(sent);
+
+//       Assert.That(rows.Count, Is.EqualTo(2));
+//       AssertBar(rows[0], 100, 101, 100, 101, 5, 2, 100.6, false);
+//       AssertBar(rows[1], 99, 99, 99, 99, 1, 1, 99, false);
+//    }
+
+//    [Test]
+//    public void TimeBars_FromBboMid_VolumeIsZero() {
+//       var schema = BboSchema();
+//       var driver = new TestDriverStream(schema);
+//       var bars = new BarBuilderStream(driver, "TIME", 60, "mid", "none");
+//       var rows = CaptureAddedRows(bars);
+
+//       var sent = new List<IBlueRow>();
+//       sent.Add(BboRow(schema, 1, T0.AddSeconds(0), "ETHUSD", 99, 101));
+//       driver.Reset(sent);
+//       sent.Add(BboRow(schema, 2, T0.AddSeconds(30), "ETHUSD", 101, 103));
+//       driver.Reset(sent);
+//       sent.Add(BboRow(schema, 3, T0.AddMinutes(1).AddSeconds(1), "ETHUSD", 100, 104));
+//       driver.Reset(sent);
+//       sent.Add(BboRow(schema, 4, T0.AddMinutes(2), "ETHUSD", 100, 102));
+//       driver.Reset(sent);
+
+//       Assert.That(rows.Count, Is.EqualTo(2));
+//       AssertBar(rows[0], 100, 102, 100, 102, 0, 2, 102, false);
+//       AssertBar(rows[1], 102, 102, 102, 102, 0, 1, 102, false);
+//    }
+
+//    [Test]
+//    public void TickBars_CloseExactlyOnNEvents() {
+//       var schema = TradeSchema();
+//       var driver = new TestDriverStream(schema);
+//       var bars = new BarBuilderStream(driver, "TICK", 3, "trade_price", "trade_size");
+//       var rows = CaptureAddedRows(bars);
+
+//       var sent = new List<IBlueRow>();
+//       sent.Add(TradeRow(schema, 1, T0, "NQ", 10, 1));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 2, T0.AddSeconds(1), "NQ", 11, 1));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 3, T0.AddSeconds(2), "NQ", 9, 1));
+//       driver.Reset(sent);
+
+//       Assert.That(rows.Count, Is.EqualTo(1));
+//       AssertBar(rows[0], 10, 11, 9, 9, 3, 3, 10, false);
+//    }
+
+//    [Test]
+//    public void VolumeBars_CloseOnThreshold() {
+//       var schema = TradeSchema();
+//       var driver = new TestDriverStream(schema);
+//       var bars = new BarBuilderStream(driver, "VOLUME", 10, "trade_price", "trade_size");
+//       var rows = CaptureAddedRows(bars);
+
+//       var sent = new List<IBlueRow>();
+//       sent.Add(TradeRow(schema, 1, T0, "ES", 100, 3));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 2, T0.AddSeconds(1), "ES", 120, 4));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 3, T0.AddSeconds(2), "ES", 110, 5));
+//       driver.Reset(sent);
+
+//       Assert.That(rows.Count, Is.EqualTo(1));
+//       AssertBar(rows[0], 100, 120, 100, 110, 12, 3, (100 * 3 + 120 * 4 + 110 * 5) / 12.0, false);
+//    }
+
+//    [Test]
+//    public void DollarBars_CloseOnThreshold() {
+//       var schema = TradeSchema();
+//       var driver = new TestDriverStream(schema);
+//       var bars = new BarBuilderStream(driver, "DOLLAR", 1000, "trade_price", "trade_size");
+//       var rows = CaptureAddedRows(bars);
+
+//       var sent = new List<IBlueRow>();
+//       sent.Add(TradeRow(schema, 1, T0, "CL", 100, 3));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 2, T0.AddSeconds(1), "CL", 120, 4));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 3, T0.AddSeconds(2), "CL", 110, 3));
+//       driver.Reset(sent);
+
+//       Assert.That(rows.Count, Is.EqualTo(1));
+//       AssertBar(rows[0], 100, 120, 100, 110, 10, 3, 111, false);
+//    }
+
+//    [Test]
+//    public void Validation_VolumeOnBboOnly_Throws() {
+//       var schema = BboSchema();
+//       var driver = new TestDriverStream(schema);
+//       var bars = new BarBuilderStream(driver, "VOLUME", 10, "mid", "trade_size");
+
+//       var row = BboRow(schema, 1, T0, "BTCUSD", 99, 101);
+//       Assert.Throws<ArgumentException>(() => {
+//          using var _ = SubscribeNoOp(bars);
+//          driver.Reset(row);
+//       });
+//    }
+
+//    [Test]
+//    public void EmitForming_EmitsFormingAndFinal() {
+//       var schema = TradeSchema();
+//       var driver = new TestDriverStream(schema);
+//       var bars = new BarBuilderStream(driver, "TICK", 2, "trade_price", "trade_size", emit_forming: true);
+//       var rows = CaptureAddedRows(bars);
+
+//       var sent = new List<IBlueRow>();
+//       sent.Add(TradeRow(schema, 1, T0, "MNQ", 10, 1));
+//       driver.Reset(sent);
+//       sent.Add(TradeRow(schema, 2, T0.AddSeconds(1), "MNQ", 12, 1));
+//       driver.Reset(sent);
+
+//       Assert.That(rows.Count, Is.EqualTo(3));
+//       Assert.That(rows[0].get<bool>("is_forming"), Is.True);
+//       Assert.That(rows[1].get<bool>("is_forming"), Is.True);
+//       Assert.That(rows[2].get<bool>("is_forming"), Is.False);
+//       AssertBar(rows[2], 10, 12, 10, 12, 2, 2, 11, false);
+//    }
+
+//    static IDisposable SubscribeNoOp(ViewStream stream) {
+//       DataStream.ChangeDelegate d = _ => { };
+//       stream.onChange += d;
+//       return new Subscription(() => stream.onChange -= d);
+//    }
+
+//    static List<IBlueRow> CaptureAddedRows(ViewStream stream) {
+//       var rows = new List<IBlueRow>();
+//       stream.onChange += o => {
+//          var dp = (IDataPacket)o;
+//          foreach (IBlueRow r in dp.addedRows) {
+//             rows.Add(new BlueRow(r));
+//          }
+//       };
+//       return rows;
+//    }
+
+//    static BlueTableSchema TradeSchema() =>
+//       new(
+//          new[] {
+//             new ColumnInfo("id", 0, typeof(int)),
+//             new ColumnInfo("ts_event", 1, typeof(string)),
+//             new ColumnInfo("contract", 2, typeof(string)),
+//             new ColumnInfo("trade_price", 3, typeof(double)),
+//             new ColumnInfo("trade_size", 4, typeof(double)),
+//          },
+//          "trade_events"
+//       );
+
+//    static BlueTableSchema BboSchema() =>
+//       new(
+//          new[] {
+//             new ColumnInfo("id", 0, typeof(int)),
+//             new ColumnInfo("ts_event", 1, typeof(string)),
+//             new ColumnInfo("contract", 2, typeof(string)),
+//             new ColumnInfo("bid_price", 3, typeof(double)),
+//             new ColumnInfo("ask_price", 4, typeof(double)),
+//          },
+//          "bbo_events"
+//       );
+
+//    static IBlueRow TradeRow(IBlueTableSchema schema, int id, DateTime ts, string contract, double px, double sz) =>
+//       new BlueRow(schema, id, ts.ToString("O"), contract, px, sz);
+
+//    static IBlueRow BboRow(IBlueTableSchema schema, int id, DateTime ts, string contract, double bid, double ask) =>
+//       new BlueRow(schema, id, ts.ToString("O"), contract, bid, ask);
+
+//    static void AssertBar(
+//       IBlueRow r,
+//       double open,
+//       double high,
+//       double low,
+//       double close,
+//       double volume,
+//       int tradeCount,
+//       double vwap,
+//       bool isForming
+//    ) {
+//       Assert.That(r.get<double>("open"), Is.EqualTo(open).Within(1e-9));
+//       Assert.That(r.get<double>("high"), Is.EqualTo(high).Within(1e-9));
+//       Assert.That(r.get<double>("low"), Is.EqualTo(low).Within(1e-9));
+//       Assert.That(r.get<double>("close"), Is.EqualTo(close).Within(1e-9));
+//       Assert.That(r.get<double>("volume"), Is.EqualTo(volume).Within(1e-9));
+//       Assert.That(r.get<int>("trade_count"), Is.EqualTo(tradeCount));
+//       Assert.That(r.get<double>("vwap"), Is.EqualTo(vwap).Within(1e-9));
+//       Assert.That(r.get<bool>("is_forming"), Is.EqualTo(isForming));
+//    }
+
+//    sealed class Subscription : IDisposable {
+//       readonly Action _dispose;
+//       public Subscription(Action dispose) => _dispose = dispose;
+//       public void Dispose() => _dispose();
+//    }
+// }

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -27,7 +27,6 @@
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
   </ItemGroup>
 
 

src/DataStreamSuite/DataStreamTests/HeikinAshiStreamTests.cs

@@ -1,121 +0,0 @@
-namespace Bts.DataStreamTests;
-
-using BCG;
-
-using BlueOrb.Operators.Signal;
-
-using Bts.DataStreamCore;
-
-[TestFixture]
-public class HeikinAshiStreamTests {
-   readonly BlueTableSchema schema = new(
-      new ColumnInfo[] {
-         new("id", 0, TypeCode.Int32),
-         new("ts_event", 1, TypeCode.DateTime),
-         new("open", 2, TypeCode.Double),
-         new("high", 3, TypeCode.Double),
-         new("low", 4, TypeCode.Double),
-         new("close", 5, TypeCode.Double),
-         new("volume", 6, TypeCode.Double),
-      }, "bars");
-
-   [Test]
-   public void AppendsHistoryAndCalculatesRecursiveHeikinAshiValues() {
-      var driver = new TestDriverStream(schema, null);
-      var heikinAshi = new HeikinAshiStream(driver);
-      var packets = new List<IDataPacket>();
-      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(2));
-
-      Assert.That(packets[0].newRows.Count, Is.EqualTo(1));
-      Assert.That(packets[0].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[0].deletedRows.Count, Is.EqualTo(0));
-
-      var first = packets[0].newRows[0];
-      Assert.That(first.get<double>("open"), Is.EqualTo(100.0).Within(1e-9));
-      Assert.That(first.get<double>("high"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(first.get<double>("low"), Is.EqualTo(90.0).Within(1e-9));
-      Assert.That(first.get<double>("close"), Is.EqualTo(105.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_open"), Is.EqualTo(102.5).Within(1e-9));
-      Assert.That(first.get<double>("ha_high"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_low"), Is.EqualTo(90.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_close"), Is.EqualTo(101.25).Within(1e-9));
-
-      Assert.That(packets[1].newRows.Count, Is.EqualTo(2));
-      Assert.That(packets[1].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[1].deletedRows.Count, Is.EqualTo(0));
-
-      var second = packets[1].newRows[1];
-      Assert.That(second.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
-      Assert.That(second.get<double>("high"), Is.EqualTo(112.0).Within(1e-9));
-      Assert.That(second.get<double>("low"), Is.EqualTo(104.0).Within(1e-9));
-      Assert.That(second.get<double>("close"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(second.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(second.get<double>("ha_high"), Is.EqualTo(112.0).Within(1e-9));
-      Assert.That(second.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(second.get<double>("ha_close"), Is.EqualTo(108.0).Within(1e-9));
-      Assert.That(second.get<double>("volume"), Is.EqualTo(20.0).Within(1e-9));
-   }
-
-   [Test]
-   public void RebuildsHistoryWhenSourceSnapshotChanges() {
-      var driver = new TestDriverStream(schema, null);
-      var heikinAshi = new HeikinAshiStream(driver);
-      var packets = new List<IDataPacket>();
-      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
-      );
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 114.0, 103.0, 111.0, 21.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(3));
-
-      Assert.That(packets[2].newRows.Count, Is.EqualTo(2));
-      Assert.That(packets[2].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[2].deletedRows.Count, Is.EqualTo(1));
-
-      var updated = packets[2].newRows[1];
-      Assert.That(updated.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
-      Assert.That(updated.get<double>("high"), Is.EqualTo(114.0).Within(1e-9));
-      Assert.That(updated.get<double>("low"), Is.EqualTo(103.0).Within(1e-9));
-      Assert.That(updated.get<double>("close"), Is.EqualTo(111.0).Within(1e-9));
-      Assert.That(updated.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(updated.get<double>("ha_high"), Is.EqualTo(114.0).Within(1e-9));
-      Assert.That(updated.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(updated.get<double>("ha_close"), Is.EqualTo(108.5).Within(1e-9));
-      Assert.That(updated.get<double>("volume"), Is.EqualTo(21.0).Within(1e-9));
-   }
-
-   static IBlueRow Row(int id, DateTime tsEvent, double open, double high, double low, double close, double volume) {
-      var row = new MutableBlueRow(
-         new BlueTableSchema(new[] {
-            new ColumnInfo("id", 0, TypeCode.Int32),
-            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
-            new ColumnInfo("open", 2, TypeCode.Double),
-            new ColumnInfo("high", 3, TypeCode.Double),
-            new ColumnInfo("low", 4, TypeCode.Double),
-            new ColumnInfo("close", 5, TypeCode.Double),
-            new ColumnInfo("volume", 6, TypeCode.Double),
-         }, "bars"));
-      row.setValue("id", id);
-      row.setValue("ts_event", tsEvent);
-      row.setValue("open", open);
-      row.setValue("high", high);
-      row.setValue("low", low);
-      row.setValue("close", close);
-      row.setValue("volume", volume);
-      return row;
-   }
-}

src/DataStreamSuite/DataStreamTests/TestDriverStream.cs

@@ -5,7 +5,7 @@ using BCG.Utilities.General;
 
 using Bts.DataStreamCore;
 
-using Spark.Core.Collections;
+using global::Spark.Core.Collections;
 
 public class TestDriverStream : ViewStream, IWorker {
    readonly IBlueTableSchema schema;

src/DataStreamSuite/DataStreamTests/VWAPStreamTests.cs

@@ -1,53 +0,0 @@
-namespace Bts.DataStreamTests;
-
-using BCG;
-
-using BlueOrb.Operators.Signal;
-
-using Bts.DataStreamCore;
-
-[TestFixture]
-public class VWAPStreamTests {
-   readonly BlueTableSchema schema = new(
-      new ColumnInfo[] {
-         new("id", 0, TypeCode.Int32),
-         new("ts_event", 1, TypeCode.DateTime),
-         new("close", 2, TypeCode.Double),
-         new("volume", 3, TypeCode.Double),
-      }, "bars");
-
-   [Test]
-   public void EmitsRowsAndCalculatesRollingVWAP() {
-      var driver = new TestDriverStream(schema, null);
-      var vwap = new VWAPStream(driver, "close", "volume", 2);
-      var packets = new List<IDataPacket>();
-      vwap.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 30, 5, DateTimeKind.Utc), 110.0, 20.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(1));
-      Assert.That(packets[0].newRows.Count, Is.EqualTo(2));
-
-      var rows = packets[0].newRows.to_list().OrderBy(r => r.get<int>("id")).ToList();
-      Assert.That(rows[0].get<double>("vwap"), Is.EqualTo(100.0).Within(1e-9));
-      Assert.That(rows[1].get<double>("vwap"), Is.EqualTo((100.0 * 10.0 + 110.0 * 20.0) / 30.0).Within(1e-9));
-   }
-
-   static IBlueRow Row(int id, DateTime tsEvent, double close, double volume) {
-      var row = new MutableBlueRow(
-         new BlueTableSchema(new[] {
-            new ColumnInfo("id", 0, TypeCode.Int32),
-            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
-            new ColumnInfo("close", 2, TypeCode.Double),
-            new ColumnInfo("volume", 3, TypeCode.Double),
-         }, "bars"));
-      row.setValue("id", id);
-      row.setValue("ts_event", tsEvent);
-      row.setValue("close", close);
-      row.setValue("volume", volume);
-      return row;
-   }
-}

src/DataStreamSuite/PersistenceServer/ConfigLoader.cs

@@ -1,33 +0,0 @@
-using System;
-using System.IO;
-using System.Text.Json;
-
-public static class ConfigLoader {
-   public static PersistenceServerConfig load() {
-      var config = new PersistenceServerConfig();
-      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.json"));
-      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.Development.json"));
-      return config;
-   }
-
-   private static void merge_from_file(PersistenceServerConfig target, string path) {
-      if (!File.Exists(path)) {
-         return;
-      }
-
-      var json = File.ReadAllText(path);
-      var loaded = JsonSerializer.Deserialize<PersistenceServerConfig>(json, new JsonSerializerOptions {
-         PropertyNameCaseInsensitive = true
-      });
-
-      if (loaded?.pub_sub is null) {
-         return;
-      }
-
-      if (target.pub_sub is null) {
-         target.pub_sub = loaded.pub_sub;
-      } else {
-         target.pub_sub.MergeWith(loaded.pub_sub);
-      }
-   }
-}

src/DataStreamSuite/PersistenceServer/PeristenceHelper.cs

@@ -1,148 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Data.SQLite;
-using System.IO;
-using System.Linq;
-
-public class PeristenceHelper {
-   private readonly string _connectionString;
-   public string database_path { get; }
-
-   public PeristenceHelper(string? databasePath = null) {
-      var resolvedPath = databasePath ?? Path.Combine(AppContext.BaseDirectory, "PersistenceServer.sqlite");
-      database_path = resolvedPath;
-      Directory.CreateDirectory(Path.GetDirectoryName(resolvedPath)!);
-      _connectionString = $"Data Source={resolvedPath};Version=3";
-   }
-
-   public void InsertRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var columns = row.Keys.Select(QuoteIdentifier).ToArray();
-      var parameterNames = row.Keys.Select(CreateParameterName).ToArray();
-      var commandText =
-         $"INSERT INTO {QuoteIdentifier(tableName)} ({string.Join(", ", columns)}) VALUES ({string.Join(", ", parameterNames)})";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      AddParameters(command, row);
-      command.ExecuteNonQuery();
-   }
-
-   public void UpdateRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var keyColumns = GetKeyColumns(row);
-      var valueColumns = row.Keys.Where(column => !keyColumns.Contains(column, StringComparer.OrdinalIgnoreCase)).ToArray();
-      if (valueColumns.Length == 0) {
-         throw new InvalidOperationException("UpdateRow requires at least one non-key column to update.");
-      }
-
-      var setClause = string.Join(", ", valueColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateParameterName(column)}"));
-      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
-      var commandText = $"UPDATE {QuoteIdentifier(tableName)} SET {setClause} WHERE {whereClause}";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      foreach (var column in valueColumns) {
-         command.Parameters.AddWithValue(CreateParameterName(column), row[column]);
-      }
-
-      foreach (var column in keyColumns) {
-         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
-      }
-
-      command.ExecuteNonQuery();
-   }
-
-   public void RemoveRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var keyColumns = GetKeyColumns(row);
-      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
-      var commandText = $"DELETE FROM {QuoteIdentifier(tableName)} WHERE {whereClause}";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      foreach (var column in keyColumns) {
-         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
-      }
-
-      command.ExecuteNonQuery();
-   }
-
-   private SQLiteConnection OpenConnection() {
-      var connection = new SQLiteConnection(_connectionString);
-      connection.Open();
-      return connection;
-   }
-
-   private static void ValidateRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      if (string.IsNullOrWhiteSpace(tableName)) {
-         throw new ArgumentException("Table name is required.", nameof(tableName));
-      }
-
-      if (row.Count == 0) {
-         throw new ArgumentException("Row must contain at least one column.", nameof(row));
-      }
-   }
-
-   private static string[] GetKeyColumns(IReadOnlyDictionary<string, string> row) {
-      if (row.ContainsKey("namespace") && row.ContainsKey("key")) {
-         return new[] { "namespace", "key" };
-      }
-
-      if (row.ContainsKey("key")) {
-         return new[] { "key" };
-      }
-
-      throw new InvalidOperationException("Row must contain 'key', or both 'namespace' and 'key', for update/delete.");
-   }
-
-   private static void EnsureTable(SQLiteConnection connection, string tableName, IEnumerable<string> columns) {
-      var existingColumns = GetExistingColumns(connection, tableName);
-      if (existingColumns.Count == 0) {
-         var createColumns = string.Join(", ", columns.Select(column => $"{QuoteIdentifier(column)} TEXT"));
-         using var createCommand =
-            new SQLiteCommand($"CREATE TABLE IF NOT EXISTS {QuoteIdentifier(tableName)} ({createColumns})", connection);
-         createCommand.ExecuteNonQuery();
-         existingColumns = GetExistingColumns(connection, tableName);
-      }
-
-      foreach (var column in columns) {
-         if (existingColumns.Contains(column)) {
-            continue;
-         }
-
-         using var alterCommand =
-            new SQLiteCommand($"ALTER TABLE {QuoteIdentifier(tableName)} ADD COLUMN {QuoteIdentifier(column)} TEXT", connection);
-         alterCommand.ExecuteNonQuery();
-      }
-   }
-
-   private static HashSet<string> GetExistingColumns(SQLiteConnection connection, string tableName) {
-      var columns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-      using var command = new SQLiteCommand($"PRAGMA table_info({QuoteIdentifier(tableName)})", connection);
-      using var reader = command.ExecuteReader();
-      while (reader.Read()) {
-         columns.Add(reader["name"].ToString()!);
-      }
-
-      return columns;
-   }
-
-   private static void AddParameters(SQLiteCommand command, IReadOnlyDictionary<string, string> row) {
-      foreach (var pair in row) {
-         command.Parameters.AddWithValue(CreateParameterName(pair.Key), pair.Value);
-      }
-   }
-
-   private static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
-
-   private static string CreateParameterName(string columnName) => $"@p_{columnName.Replace("\"", string.Empty)}";
-
-   private static string CreateWhereParameterName(string columnName) => $"@w_{columnName.Replace("\"", string.Empty)}";
-}

src/DataStreamSuite/PersistenceServer/PersistenceCallHandler.cs

@@ -1,155 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Net;
-using System.Text.Json;
-
-public class PersistenceCallHandler {
-   private const string KeyValueTableName = "key_value_store";
-   private readonly PeristenceHelper _persistenceHelper;
-   private readonly PubSubSync _pub_sub_sync;
-
-   public PersistenceCallHandler(PeristenceHelper persistenceHelper, PubSubSync pub_sub_sync) {
-      _persistenceHelper = persistenceHelper;
-      _pub_sub_sync = pub_sub_sync;
-   }
-
-   public void Handle(HttpListenerRequest request, string payload) {
-      Console.WriteLine("----- Incoming REST call -----");
-      Console.WriteLine($"Timestamp: {DateTimeOffset.Now:O}");
-      Console.WriteLine($"Method: {request.HttpMethod}");
-      Console.WriteLine($"Url: {request.Url}");
-      Console.WriteLine($"RawUrl: {request.RawUrl}");
-      Console.WriteLine($"RemoteEndPoint: {request.RemoteEndPoint}");
-      Console.WriteLine($"UserAgent: {request.UserAgent}");
-      Console.WriteLine($"ContentType: {request.ContentType}");
-      Console.WriteLine($"ContentLength: {request.ContentLength64}");
-
-      if (request.QueryString.Count > 0) {
-         Console.WriteLine("Query:");
-         foreach (var key in request.QueryString.AllKeys) {
-            if (key is null) {
-               continue;
-            }
-
-            Console.WriteLine($"  {key} = {request.QueryString[key]}");
-         }
-      }
-
-      if (request.Headers.Count > 0) {
-         Console.WriteLine("Headers:");
-         foreach (var headerKey in request.Headers.AllKeys) {
-            if (headerKey is null) {
-               continue;
-            }
-
-            Console.WriteLine($"  {headerKey}: {request.Headers[headerKey]}");
-         }
-      }
-
-      Console.WriteLine("Body:");
-      Console.WriteLine(string.IsNullOrWhiteSpace(payload) ? "  <empty>" : payload);
-
-      var rpcRequest = ParseRpcRequest(payload);
-      HandleRpc(rpcRequest.FunctionName, rpcRequest.Arguments);
-      Console.WriteLine("------------------------------");
-   }
-
-   private static PersistenceRpcRequest ParseRpcRequest(string payload) {
-      if (string.IsNullOrWhiteSpace(payload)) {
-         throw new InvalidOperationException("RPC payload is empty.");
-      }
-
-      using var document = JsonDocument.Parse(payload);
-      var root = document.RootElement;
-
-      if (!root.TryGetProperty("function_name", out var functionNameElement) ||
-          functionNameElement.ValueKind != JsonValueKind.String) {
-         throw new InvalidOperationException("RPC payload must contain a string 'function_name'.");
-      }
-
-      if (!root.TryGetProperty("arguments", out var argumentsElement) ||
-          argumentsElement.ValueKind != JsonValueKind.Array) {
-         throw new InvalidOperationException("RPC payload must contain an 'arguments' array.");
-      }
-
-      var arguments = new List<string>();
-      foreach (var argument in argumentsElement.EnumerateArray()) {
-         if (argument.ValueKind != JsonValueKind.String) {
-            throw new InvalidOperationException("RPC arguments must all be strings.");
-         }
-
-         arguments.Add(argument.GetString()!);
-      }
-
-      return new PersistenceRpcRequest(functionNameElement.GetString()!, arguments.ToArray());
-   }
-
-   private void HandleRpc(string functionName, params string[] arguments) {
-      switch (functionName) {
-         case "keyvalue_insert":
-            RequireArgumentCount(functionName, arguments, 3);
-            HandleKeyValueInsert(arguments[0], arguments[1], arguments[2]);
-            break;
-         case "keyvalue_update":
-            RequireArgumentCount(functionName, arguments, 3);
-            HandleKeyValueUpdate(arguments[0], arguments[1], arguments[2]);
-            break;
-         case "keyvalue_delete":
-            RequireArgumentCount(functionName, arguments, 2);
-            HandleKeyValueDelete(arguments[0], arguments[1]);
-            break;
-         default:
-            throw new InvalidOperationException($"Unknown RPC function '{functionName}'.");
-      }
-   }
-
-   private static void RequireArgumentCount(string functionName, string[] arguments, int expectedCount) {
-      if (arguments.Length != expectedCount) {
-         throw new InvalidOperationException(
-            $"RPC function '{functionName}' expected {expectedCount} arguments but received {arguments.Length}.");
-      }
-   }
-
-   private void HandleKeyValueInsert(string itemNamespace, string key, string value) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-         ["value"] = value,
-      };
-      _persistenceHelper.InsertRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_insert(namespace='{itemNamespace}', key='{key}', value='{value}')");
-   }
-
-   private void HandleKeyValueUpdate(string itemNamespace, string key, string value) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-         ["value"] = value,
-      };
-      _persistenceHelper.UpdateRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_update(namespace='{itemNamespace}', key='{key}', value='{value}')");
-   }
-
-   private void HandleKeyValueDelete(string itemNamespace, string key) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-      };
-      _persistenceHelper.RemoveRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_delete(namespace='{itemNamespace}', key='{key}')");
-   }
-}
-
-public record PersistenceRpcRequest {
-   public PersistenceRpcRequest(string functionName, string[] arguments) {
-      FunctionName = functionName;
-      Arguments = arguments;
-      Console.WriteLine($"PersistenceRpcRequest created for function '{FunctionName}' with {Arguments.Length} argument(s).");
-   }
-
-   public string FunctionName { get; init; }
-   public string[] Arguments { get; init; }
-}

src/DataStreamSuite/PersistenceServer/PersistenceServer.csproj

@@ -1,23 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Content Include="appsettings.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/PersistenceServer/PersistenceServerConfig.cs

@@ -1,5 +0,0 @@
-using Bts.PubSub;
-
-public class PersistenceServerConfig {
-   public PubSubConfig? pub_sub { get; set; }
-}

src/DataStreamSuite/PersistenceServer/Program.cs

@@ -1,113 +0,0 @@
-using Bts.DataStreamCore;
-using BCG;
-
-using System;
-using System.IO;
-using System.Net;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-
-const int Port = 5079;
-var prefix = $"http://localhost:{Port}/";
-
-var listener = new HttpListener();
-listener.Prefixes.Add(prefix);
-var config = ConfigLoader.load();
-var persistenceHelper = new PeristenceHelper();
-var pub_sub_sync = new PubSubSync(persistenceHelper.database_path, config.pub_sub);
-var callHandler = new PersistenceCallHandler(persistenceHelper, pub_sub_sync);
-
-Console.WriteLine("PersistenceServer starting.");
-Console.WriteLine($"Listening on {prefix}");
-Console.WriteLine($"DataStreamCore loaded from: {typeof(IDataPacket).Assembly.FullName}");
-Console.WriteLine("Press Ctrl+C to stop.");
-
-var shutdown = new CancellationTokenSource();
-
-Console.CancelKeyPress += (_, eventArgs) => {
-   eventArgs.Cancel = true;
-   Console.WriteLine("Shutdown requested.");
-   shutdown.Cancel();
-   if (listener.IsListening) {
-      listener.Stop();
-   }
-};
-
-try {
-   pub_sub_sync.publish_all_tables();
-   listener.Start();
-   await RunAsync(listener, callHandler, shutdown.Token);
-   return 0;
-} catch (HttpListenerException ex) when (shutdown.IsCancellationRequested) {
-   Console.WriteLine($"Listener stopped: {ex.Message}");
-   return 0;
-} catch (Exception ex) {
-   Console.Error.WriteLine($"PersistenceServer crashed: {ex}");
-   return 1;
-} finally {
-   listener.Close();
-}
-
-static async Task RunAsync(
-   HttpListener listener,
-   PersistenceCallHandler callHandler,
-   CancellationToken cancellationToken) {
-   while (!cancellationToken.IsCancellationRequested) {
-      HttpListenerContext context;
-      try {
-         context = await listener.GetContextAsync();
-      } catch (HttpListenerException) when (cancellationToken.IsCancellationRequested) {
-         break;
-      } catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested) {
-         break;
-      }
-
-      _ = Task.Run(() => HandleRequestAsync(context, callHandler, cancellationToken), cancellationToken);
-   }
-}
-
-static async Task HandleRequestAsync(
-   HttpListenerContext context,
-   PersistenceCallHandler callHandler,
-   CancellationToken cancellationToken) {
-   var request = context.Request;
-   var response = context.Response;
-
-   try {
-      var body = await ReadBodyAsync(request, cancellationToken);
-      callHandler.Handle(request, body);
-
-      var responseText =
-         $$"""
-         {"status":"ok","method":"{{request.HttpMethod}}","path":"{{request.RawUrl ?? "/"}}"}
-         """;
-
-      var bytes = Encoding.UTF8.GetBytes(responseText);
-      response.StatusCode = (int)HttpStatusCode.OK;
-      response.ContentType = "application/json";
-      response.ContentEncoding = Encoding.UTF8;
-      response.ContentLength64 = bytes.LongLength;
-      await response.OutputStream.WriteAsync(bytes, cancellationToken);
-   } catch (Exception ex) {
-      Console.Error.WriteLine($"Request handling failed: {ex}");
-      response.StatusCode = (int)HttpStatusCode.InternalServerError;
-   } finally {
-      response.OutputStream.Close();
-      response.Close();
-   }
-}
-
-static async Task<string> ReadBodyAsync(HttpListenerRequest request, CancellationToken cancellationToken) {
-   if (!request.HasEntityBody) {
-      return string.Empty;
-   }
-
-   using var reader = new StreamReader(
-      request.InputStream,
-      request.ContentEncoding ?? Encoding.UTF8,
-      detectEncodingFromByteOrderMarks: true,
-      leaveOpen: true);
-
-   return await reader.ReadToEndAsync(cancellationToken);
-}