Adding logs to CME MD

To mount a custom entrypoint script, put it in the config profile dir
and define the following env vars:

    SPARKD__IMAGE=sparkd_symbols
    SPARKD__MAIN_ENTRYPOINT=/home/bts/.local/share/bts/sparkd/conf/<custom-entrypoint>

To invoke the "normal" sparkd entrypoint script from the mounted one:

    ./sparkd-entrypoint.sh

.devcontainer/devcontainer.json

@@ -1,5 +1,11 @@
 {
-  "dockerFile": "Dockerfile",
+  "build": {
+    "dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
+    "options": [
+      "--secret",
+      "id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
+    ]
+  },
   "hostRequirements": {
     "memory": "8gb",
     "storage": "10gb"

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.env

@@ -29,6 +29,9 @@
 #     tools/deploy.sh --help
 #
 
+### Host/Runtime ###
+TZ=Etc/UTC
+
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
 #   src/server/sparkd/Dockerfile
@@ -38,6 +41,9 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
+# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
+# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
+# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
@@ -45,6 +51,8 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
+SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
+SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
 
 ### PubSubServer ###
 PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development

.vscode/launch.json

@@ -142,14 +142,14 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "preLaunchTask": "Build DataStream (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
           "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -164,8 +164,8 @@
         "program": "DataStreamQueryAnalyzer.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
           "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -183,10 +183,6 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
       "env": {
         "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
@@ -212,6 +208,43 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "Sdk Demo",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "preLaunchTask": "Build Api Server (Debug)",
+      "windows": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "linux": {
+        "program": "Bts.Spark.Sdk.Demo",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sdk Demo (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "linux": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
     {
       "name": "Spark",
       "type": "coreclr",
@@ -300,6 +333,7 @@
       "program": "",
       "linux": {
         "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "preLaunchTask": "Build Sparkd (Debug | Clang)",
       },
       "args": [
         "run"
@@ -349,6 +383,25 @@
         }
       ]
     },
+    {
+      "name": "Sparkd Attach",
+      "type": "cppdbg",
+      "request": "attach",
+      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
   ],
   "inputs": [
     {
@@ -374,37 +427,6 @@
       ],
       "default": "MarketDataPublisher",
     },
-    {
-      "id": "dataStreamPublisherEnvironment",
-      "type": "pickString",
-      "description": "DataStreamPublisher Configuration",
-      "options": [
-        "Local",
-        "ReplayMarketData",
-      ],
-      "default": "Local",
-    },
-    {
-      "id": "pubSubServerConfig",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
-      ],
-      "default": "PubSubServer",
-    },
-    {
-      "id": "pubSubServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "dataStreamQueryAnalyzerEnvironment",
       "type": "pickString",
@@ -416,12 +438,34 @@
       "default": "Development",
     },
     {
-      "id": "sparkEnvironment",
+      "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
-      "description": "Spark Configuration",
+      "description": "DataStreamPublisher Configuration",
+      "options": [
+        "Local",
+        "ReplayMarketData",
+      ],
+      "default": "Local",
+    },
+    {
+      "id": "pubSubServerEnvironment",
+      "type": "pickString",
+      "description": "PubSubServer Configuration",
+      "options": [
+        "Development",
+        "EchoServer",
+        "Local",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "sdkDemoEnvironment",
+      "type": "pickString",
+      "description": "Sdk Demo Configuration",
       "options": [
         "Development",
         "Local",
+        "Staging",
       ],
       "default": "Development",
     },
@@ -431,6 +475,16 @@
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
+    {
+      "id": "sparkEnvironment",
+      "type": "pickString",
+      "description": "Spark Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkTest",
       "type": "promptString",

.vscode/tasks.json

@@ -121,57 +121,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Build DataStreamQueryAnalyzer",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Build Spark",
       "group": "build",
@@ -223,6 +172,66 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Build Spark Sdk",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark Sdk (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark Sdk (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Build Sparkd",
       "group": "build",
@@ -527,17 +536,16 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStream",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -547,17 +555,16 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -567,17 +574,16 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStream (Release)",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -1039,6 +1045,16 @@
       ],
       "default": "Debug",
     },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkdCmakePreset",
       "type": "pickString",
@@ -1081,7 +1097,6 @@
       "options": [
         "Development",
         "EchoServer",
-        "Local",
         "Production",
         "ProductionAws",
         "Staging",

Bts.Spark.Sdk.sln

@@ -0,0 +1,102 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.Build.0 = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.Build.0 = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.Build.0 = Release|x64
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.Build.0 = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+	EndGlobalSection
+EndGlobal

CMakePresets.json

@@ -8,14 +8,31 @@
 
   "configurePresets": [
     {
-      "name": "spark",
+      "name": "spark-base",
+      "hidden": true,
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    },
+
+    {
+      "name": "spark-windows",
+      "inherits": "spark-base",
+      "displayName": "Spark",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
@@ -23,18 +40,9 @@
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
       }
     },
 
@@ -151,17 +159,17 @@
     },
 
     {
-      "name": "spark-api-server-windows",
-      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)"
-    },
-
-    {
-      "name": "spark-api-server-linux",
-      "inherits": "sparkd",
-      "displayName": "Spark Api Server (Linux)",
+      "name": "spark-linux",
+      "inherits": "spark-base",
+      "displayName": "Spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
+        "platform": "linux",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -170,21 +178,26 @@
           "type": "BOOL",
           "value": "OFF"
         },
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
         "CMAKE_EXPORT_COMPILE_COMMANDS": {
           "type": "BOOL",
           "value": "ON"
         },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "EXPORT_MD_CONSUMER": {
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
+        "DATABENTO_USE_EXTERNAL_JSON": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "EXTERNALS_ROOT": "${sourceDir}/externals/linux"
       }
     },
 
@@ -204,87 +217,32 @@
       "cacheVariables": {
         "CCAPI_LOG_LEVEL": "Trace"
       }
-    },
-
-    {
-      "name": "data-stream-suite-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
-      "inherits": "spark",
-      "displayName": "Data Stream Suite (Windows)",
-      "description": "Data Stream utils ported over from Edge",
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
-      }
-    },
-
-    {
-      "name": "data-stream-suite-linux",
-      "displayName": "Data Stream Suite (Linux)",
-      "description": "Data Stream utils ported over from Edge",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
-      "inherits": "sparkd",
-      "generator": "Ninja Multi-Config",
-      "cacheVariables": {
-        "BUILD_ADMIN_WEB_APP": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SDK": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SPARKD": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_C_COMPILER": "clang",
-        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
-      }
     }
   ],
 
   "buildPresets": [
     {
-      "name": "spark",
+      "name": "spark-windows",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
-      "hidden": true
+      "hidden": true,
+      "configurePreset": "spark-windows"
     },
 
     {
-      "name": "spark-debug",
+      "name": "spark-windows-debug",
       "displayName": "Spark (Debug)",
-      "inherits": "spark",
-      "configurePreset": "spark",
+      "inherits": "spark-windows",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-release",
+      "name": "spark-windows-release",
       "displayName": "Spark (Release)",
-      "inherits": "spark",
-      "configurePreset": "spark",
+      "inherits": "spark-windows",
       "configuration": "Release"
     },
 
@@ -327,52 +285,27 @@
     },
 
     {
-      "name": "spark-api-server-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
-      "hidden": true,
-      "configurePreset": "spark-api-server-windows"
-    },
-
-    {
-      "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Windows | Debug)",
-      "inherits": "spark-api-server-windows",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Windows | Release)",
-      "inherits": "spark-api-server-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "spark-api-server-linux",
+      "name": "spark-linux",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
       "hidden": true,
-      "configurePreset": "spark-api-server-linux"
+      "configurePreset": "spark-linux"
     },
 
     {
-      "name": "spark-api-server-linux-debug",
-      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Debug)",
+      "name": "spark-linux-debug",
+      "inherits": "spark-linux",
+      "displayName": "Spark (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-api-server-linux-release",
-      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Release)",
+      "name": "spark-linux-release",
+      "inherits": "spark-linux",
+      "displayName": "Spark (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -389,34 +322,6 @@
       "configurePreset": "sparkd-ccapi-diag",
       "displayName": "Debug (ccapi diag)",
       "inherits": "sparkd"
-    },
-
-    {
-      "name": "data-stream-suite-windows-debug",
-      "displayName": "Data Stream Suite (Windows | Debug)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-windows-release",
-      "displayName": "Data Stream Suite (Windows | Release)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "data-stream-suite-linux-debug",
-      "displayName": "Data Stream Suite (Linux | Debug)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-linux-release",
-      "displayName": "Data Stream Suite (Linux | Release)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Release"
     }
   ]
 }

Spark.Api.Server.sln

@@ -1,10 +1,11 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
@@ -42,180 +43,538 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sdk", "sdk", "{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\sdk\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}"
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
+		MinSizeRel|Any CPU = MinSizeRel|Any CPU
+		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
+		Release|Any CPU = Release|Any CPU
+		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
+		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
+		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.Build.0 = Debug|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.ActiveCfg = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.Build.0 = Release|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.Build.0 = Debug|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.ActiveCfg = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.Build.0 = Release|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.Build.0 = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.Build.0 = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.Build.0 = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.Build.0 = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.Build.0 = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.Build.0 = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.Build.0 = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.Build.0 = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.Build.0 = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.Build.0 = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.Build.0 = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.Build.0 = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.Build.0 = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.Build.0 = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.Build.0 = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.Build.0 = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.Build.0 = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.Build.0 = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.Build.0 = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.Build.0 = Release|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.Build.0 = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.Build.0 = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.ActiveCfg = Release|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.Build.0 = Release|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
@@ -227,10 +586,17 @@ Global
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184} = {56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
 	EndGlobalSection
 EndGlobal

cmake/Protobuf.cmake

@@ -1,182 +0,0 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
-set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
-
-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
-      DEPENDS ${ABS_FIL}
-      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
-
-    set(TEMPLATES
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
-    )
-
-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
-      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
-        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
-        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE}
-          ARGS
-            --cpp_out=${_ARGS_CXX_OUT_DIR}
-            --plugin=protoc-gen-service=${_plugin_cmd}
-            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
-            -I ${PROTOBUF_INCLUDE_DIRS}
-            -I ${_protobuf_src_dir}
-            ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_cmd}
-          ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
-    else()
-      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
-        ARGS
-          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
-    endif()
-  endforeach()
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-

compose.yaml

@@ -2,6 +2,7 @@ services:
   sparkd:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -9,24 +10,35 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-shell:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+    build:
+      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: "do sleep 1; done"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -42,6 +54,7 @@ services:
   sparkd-symbols:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
       - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
@@ -50,26 +63,8 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/.local/share/bts/sparkd/conf/
-        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -77,6 +72,33 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+  sparkd-multicast:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-2048mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       restart_policy:
         condition: any
@@ -84,16 +106,15 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-
   sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
+    image: bluetradingsystems/linux_dotnet8
     volumes:
       - type: volume
         read_only: true
         source: sparkd_data
         target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
         volume:
-          subpath: logging
+          subpath: logging.sparkd
     command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
 
   sparkd-ping-test:
@@ -104,14 +125,14 @@ services:
     build:
       context: .
       dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
+        FROM bluetradingsystems/linux_dotnet8
         USER "root"
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
 
   pubsub:
     volumes:
-      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/PubSubServer/data
+      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
     build:
       dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
       args:
@@ -121,20 +142,20 @@ services:
       - 22019:22019
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/PubSubServer
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" ]; then
-          mv "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
+        if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
+          mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env'
+        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
           set -a
-          . /home/bts/.local/share/bts/PubSubServer/conf/.env
+          . /home/bts/.local/share/bts/pubsub/conf/.env
           set +a
         fi
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env.local'
+        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
           set -a
-          . /home/bts/.local/share/bts/PubSubServer/conf/.env.local
+          . /home/bts/.local/share/bts/pubsub/conf/.env.local
           set +a
         fi
         ./PubSubServer"
@@ -144,8 +165,8 @@ services:
 
   md-publisher:
     volumes:
-      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/DataStreamPublisher/data
-      - historic_md:/home/bts/.local/share/bts/databento
+      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
+      - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
     build:
       dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
       args:
@@ -153,20 +174,20 @@ services:
         - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/DataStreamPublisher
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" ]; then
-          mv "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
+        if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
+          mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env'
+        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
           set -a
-          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env
+          . /home/bts/.local/share/bts/md_publisher/conf/.env
           set +a
         fi
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local'
+        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
           set -a
-          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local
+          . /home/bts/.local/share/bts/md_publisher/conf/.env.local
           set +a
         fi
       dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
@@ -195,3 +216,6 @@ volumes:
     name: "md_publisher_22019"
   historic_md:
     name: "historic_md"
+  md_recording:
+    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
+    external: true

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -0,0 +1,39 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+RUN apt-get update && apt-get install -y tzdata
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet8
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile

@@ -55,14 +55,16 @@ RUN \
    clangd-$clang_major
 
 # CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
+RUN set -eux \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends ca-certificates curl gpg \
+  && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
+    | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
+  && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    > /etc/apt/sources.list.d/kitware.list \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends cmake \
+  && rm -rf /var/lib/apt/lists/*
 
 # Dotnet
 RUN apt-get update && apt-get install -y dotnet8
@@ -80,9 +82,14 @@ RUN mkdir -p /etc/apt/keyrings \
 # Emumba (deps)
 RUN apt-get update && apt-get -y install rapidjson-dev
 # Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
+ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
+RUN --mount=type=secret,id=bitbucket_access_token \
+  set -eux \
+    && apt-get update \
+    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
+    && curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
+    && apt-get install -y /tmp/emumba-deribit-connector.deb \
+    && rm -f /tmp/emumba-deribit-connector.deb
 
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -90,6 +97,18 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
+# Rakurai
+ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
+RUN --mount=type=secret,id=bitbucket_access_token \
+  set -eux \
+    && apt-get update \
+    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
+    && curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
+    # Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
+    # spdlog headers.
+    && apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
+    && rm -f /tmp/rakuirai.deb
+
 # Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \

docker/images/bluetradingsystems/linux_shell/Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:24.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+  fish \
+  less \
+  vim
+
+USER bts
+WORKDIR /home/bts

global.json

@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "6.0.200",
+    "rollForward": "latestMajor"
+  }
+}

requirements.txt

@@ -18,10 +18,10 @@ MarkupSafe==3.0.2
 mccabe==0.6.1
 numpy==2.2.6
 platformdirs==2.3.0
-protobuf==4.21.12
+protobuf==6.33.4
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
+pybind11==3.0.1
+pybind11-stubgen==2.5.5
 pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20

spark_client.sln

@@ -1,164 +1,306 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
-		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
-	EndGlobalSection
-EndGlobal
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+	EndGlobalSection
+EndGlobal

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -11,10 +11,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ArithmeticExpression">
     <Scanner Include="ArithmeticExpressionTokens.lex" />

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -5,9 +5,5 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
+  
 </Project>

src/DataStreamSuite/BlueOrbDSL/DSQA_LAST_EXECUTED_QUERY.txt

@@ -0,0 +1 @@
+r=TimeSeriesStream(ReplayFutureLiveBBOStream("ES"))

src/DataStreamSuite/BlueOrbDSL/DSQA_RESULTS_EXPORT_20260216_195816.csv

@@ -0,0 +1,3 @@
+x,y,z,id
+123,1.001,abc,1
+123,1.001,abc,0

src/DataStreamSuite/BlueOrbDSL/DSQA_RESULTS_EXPORT_20260216_195818.csv

@@ -0,0 +1,3 @@
+x,y,z,id
+123,1.001,abc,1
+123,1.001,abc,0

src/DataStreamSuite/BlueOrbDSL/DSQA_RESULTS_EXPORT_20260313_055725.csv

@@ -0,0 +1,78 @@
+id,ts_event,contract,bid_price,bid_size,ask_price,ask_size,timestamp
+76,2026-03-13T05:57:25.177965500Z,ESU5,6363.75,51,6364,63,3/13/2026 5:57:25 AM
+75,2026-03-13T05:57:25.068062900Z,ESU5,6363.75,52,6364,63,3/13/2026 5:57:25 AM
+74,2026-03-13T05:57:24.963428500Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+73,2026-03-13T05:57:24.781215700Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+72,2026-03-13T05:57:24.676290100Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+71,2026-03-13T05:57:24.566898400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+70,2026-03-13T05:57:24.430800400Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+69,2026-03-13T05:57:24.273683400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+68,2026-03-13T05:57:24.167656100Z,ESU5,6363.75,56,6364,64,3/13/2026 5:57:24 AM
+67,2026-03-13T05:57:23.995720100Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:23 AM
+66,2026-03-13T05:57:23.894029100Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
+65,2026-03-13T05:57:23.728897300Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
+64,2026-03-13T05:57:23.614316800Z,ESU5,6363.75,52,6364,68,3/13/2026 5:57:23 AM
+63,2026-03-13T05:57:23.503752700Z,ESU5,6363.75,51,6364,68,3/13/2026 5:57:23 AM
+62,2026-03-13T05:57:23.289108200Z,ESU5,6363.75,50,6364,68,3/13/2026 5:57:23 AM
+61,2026-03-13T05:57:23.155673400Z,ESU5,6363.75,49,6364,68,3/13/2026 5:57:23 AM
+60,2026-03-13T05:57:23.028898400Z,ESU5,6363.75,48,6364,68,3/13/2026 5:57:23 AM
+59,2026-03-13T05:57:22.887132700Z,ESU5,6363.75,46,6364,68,3/13/2026 5:57:22 AM
+58,2026-03-13T05:57:22.676916300Z,ESU5,6363.75,44,6364,68,3/13/2026 5:57:22 AM
+57,2026-03-13T05:57:22.520328000Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:22 AM
+56,2026-03-13T05:57:22.405244600Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:22 AM
+55,2026-03-13T05:57:22.294674100Z,ESU5,6363.75,41,6364,69,3/13/2026 5:57:22 AM
+54,2026-03-13T05:57:22.163467900Z,ESU5,6363.75,38,6364,69,3/13/2026 5:57:22 AM
+53,2026-03-13T05:57:21.992443000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
+52,2026-03-13T05:57:21.872338400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
+51,2026-03-13T05:57:21.742801800Z,ESU5,6363.75,37,6364,71,3/13/2026 5:57:21 AM
+50,2026-03-13T05:57:21.557711100Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
+49,2026-03-13T05:57:21.434977000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
+48,2026-03-13T05:57:21.285751600Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
+47,2026-03-13T05:57:21.184392000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
+46,2026-03-13T05:57:21.076843800Z,ESU5,6363.75,34,6364,73,3/13/2026 5:57:21 AM
+45,2026-03-13T05:57:20.894131800Z,ESU5,6363.75,32,6364,73,3/13/2026 5:57:20 AM
+44,2026-03-13T05:57:20.759550000Z,ESU5,6363.75,30,6364,73,3/13/2026 5:57:20 AM
+43,2026-03-13T05:57:20.641547900Z,ESU5,6363.75,28,6364,73,3/13/2026 5:57:20 AM
+42,2026-03-13T05:57:20.540293200Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
+41,2026-03-13T05:57:20.439034600Z,ESU5,6363.75,27,6364,74,3/13/2026 5:57:20 AM
+40,2026-03-13T05:57:20.337795000Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
+39,2026-03-13T05:57:20.236606600Z,ESU5,6363.75,27,6364,72,3/13/2026 5:57:20 AM
+38,2026-03-13T05:57:20.135417700Z,ESU5,6363.75,30,6364,72,3/13/2026 5:57:20 AM
+37,2026-03-13T05:57:20.034272700Z,ESU5,6363.75,32,6364,72,3/13/2026 5:57:20 AM
+36,2026-03-13T05:57:19.933096800Z,ESU5,6363.75,34,6364,72,3/13/2026 5:57:19 AM
+35,2026-03-13T05:57:19.832012600Z,ESU5,6363.75,36,6364,72,3/13/2026 5:57:19 AM
+34,2026-03-13T05:57:19.730878500Z,ESU5,6363.75,38,6364,72,3/13/2026 5:57:19 AM
+33,2026-03-13T05:57:19.629787400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
+32,2026-03-13T05:57:19.528697000Z,ESU5,6363.75,38,6364,70,3/13/2026 5:57:19 AM
+31,2026-03-13T05:57:19.427077000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
+30,2026-03-13T05:57:19.326450300Z,ESU5,6363.75,39,6364,71,3/13/2026 5:57:19 AM
+29,2026-03-13T05:57:19.225427900Z,ESU5,6363.75,39,6364,70,3/13/2026 5:57:19 AM
+28,2026-03-13T05:57:19.123339800Z,ESU5,6363.75,40,6364,70,3/13/2026 5:57:19 AM
+27,2026-03-13T05:57:19.021936500Z,ESU5,6363.75,41,6364,70,3/13/2026 5:57:19 AM
+26,2026-03-13T05:57:18.920859200Z,ESU5,6363.75,45,6364,70,3/13/2026 5:57:18 AM
+25,2026-03-13T05:57:18.820148400Z,ESU5,6363.75,46,6364,70,3/13/2026 5:57:18 AM
+24,2026-03-13T05:57:18.718988100Z,ESU5,6363.75,46,6364,69,3/13/2026 5:57:18 AM
+23,2026-03-13T05:57:18.618299700Z,ESU5,6363.75,49,6364,69,3/13/2026 5:57:18 AM
+22,2026-03-13T05:57:18.517138100Z,ESU5,6363.75,52,6364,69,3/13/2026 5:57:18 AM
+21,2026-03-13T05:57:18.416335200Z,ESU5,6363.75,53,6364,69,3/13/2026 5:57:18 AM
+20,2026-03-13T05:57:18.315335900Z,ESU5,6363.75,55,6364,69,3/13/2026 5:57:18 AM
+19,2026-03-13T05:57:18.214368600Z,ESU5,6363.75,57,6364,69,3/13/2026 5:57:18 AM
+18,2026-03-13T05:57:18.113754700Z,ESU5,6363.75,57,6364,65,3/13/2026 5:57:18 AM
+17,2026-03-13T05:57:18.012706500Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:18 AM
+16,2026-03-13T05:57:17.911672600Z,ESU5,6363.75,54,6364,65,3/13/2026 5:57:17 AM
+15,2026-03-13T05:57:17.810879900Z,ESU5,6363.75,52,6364,65,3/13/2026 5:57:17 AM
+14,2026-03-13T05:57:17.710089700Z,ESU5,6363.75,49,6364,65,3/13/2026 5:57:17 AM
+13,2026-03-13T05:57:17.609388300Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+12,2026-03-13T05:57:17.508697400Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+11,2026-03-13T05:57:17.408027000Z,ESU5,6363.75,53,6364,67,3/13/2026 5:57:17 AM
+10,2026-03-13T05:57:17.307406200Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+9,2026-03-13T05:57:17.206714300Z,ESU5,6363.75,49,6364,71,3/13/2026 5:57:17 AM
+8,2026-03-13T05:57:17.105530300Z,ESU5,6363.75,48,6364,71,3/13/2026 5:57:17 AM
+7,2026-03-13T05:57:17.004796800Z,ESU5,6363.75,47,6364,71,3/13/2026 5:57:17 AM
+6,2026-03-13T05:57:16.904113900Z,ESU5,6363.75,46,6364,71,3/13/2026 5:57:16 AM
+5,2026-03-13T05:57:16.803381900Z,ESU5,6363.75,43,6364,71,3/13/2026 5:57:16 AM
+4,2026-03-13T05:57:16.701969600Z,ESU5,6363.75,43,6364,72,3/13/2026 5:57:16 AM
+3,2026-03-13T05:57:16.600765600Z,ESU5,6363.75,43,6364,73,3/13/2026 5:57:16 AM
+2,2026-03-13T05:57:16.499986300Z,ESU5,6363.75,43,6364,74,3/13/2026 5:57:16 AM
+1,2026-03-13T05:57:16.399241700Z,ESU5,6363.75,43,6364,75,3/13/2026 5:57:16 AM
+0,2026-03-13T05:57:16.264038000Z,ESU5,6363.75,43,6364,76,3/13/2026 5:57:16 AM

src/DataStreamSuite/BlueOrbDSL/DSQA_RESULTS_EXPORT_20260313_055727.csv

@@ -0,0 +1,91 @@
+id,ts_event,contract,bid_price,bid_size,ask_price,ask_size,timestamp
+89,2026-03-13T05:57:26.904401600Z,ESU5,6363.75,43,6364,67,3/13/2026 5:57:26 AM
+88,2026-03-13T05:57:26.778037200Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:26 AM
+87,2026-03-13T05:57:26.597038500Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:26 AM
+86,2026-03-13T05:57:26.438768300Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:26 AM
+85,2026-03-13T05:57:26.328259100Z,ESU5,6363.75,43,6364,67,3/13/2026 5:57:26 AM
+84,2026-03-13T05:57:26.216211200Z,ESU5,6363.75,44,6364,67,3/13/2026 5:57:26 AM
+83,2026-03-13T05:57:26.099281200Z,ESU5,6363.75,46,6364,67,3/13/2026 5:57:26 AM
+82,2026-03-13T05:57:25.979814900Z,ESU5,6363.75,48,6364,67,3/13/2026 5:57:25 AM
+81,2026-03-13T05:57:25.852317700Z,ESU5,6363.75,48,6364,66,3/13/2026 5:57:25 AM
+80,2026-03-13T05:57:25.684323100Z,ESU5,6363.75,48,6364,65,3/13/2026 5:57:25 AM
+79,2026-03-13T05:57:25.539436900Z,ESU5,6363.75,48,6364,64,3/13/2026 5:57:25 AM
+78,2026-03-13T05:57:25.425439200Z,ESU5,6363.75,49,6364,64,3/13/2026 5:57:25 AM
+77,2026-03-13T05:57:25.298054100Z,ESU5,6363.75,51,6364,64,3/13/2026 5:57:25 AM
+76,2026-03-13T05:57:25.177965500Z,ESU5,6363.75,51,6364,63,3/13/2026 5:57:25 AM
+75,2026-03-13T05:57:25.068062900Z,ESU5,6363.75,52,6364,63,3/13/2026 5:57:25 AM
+74,2026-03-13T05:57:24.963428500Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+73,2026-03-13T05:57:24.781215700Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+72,2026-03-13T05:57:24.676290100Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+71,2026-03-13T05:57:24.566898400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+70,2026-03-13T05:57:24.430800400Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
+69,2026-03-13T05:57:24.273683400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
+68,2026-03-13T05:57:24.167656100Z,ESU5,6363.75,56,6364,64,3/13/2026 5:57:24 AM
+67,2026-03-13T05:57:23.995720100Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:23 AM
+66,2026-03-13T05:57:23.894029100Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
+65,2026-03-13T05:57:23.728897300Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
+64,2026-03-13T05:57:23.614316800Z,ESU5,6363.75,52,6364,68,3/13/2026 5:57:23 AM
+63,2026-03-13T05:57:23.503752700Z,ESU5,6363.75,51,6364,68,3/13/2026 5:57:23 AM
+62,2026-03-13T05:57:23.289108200Z,ESU5,6363.75,50,6364,68,3/13/2026 5:57:23 AM
+61,2026-03-13T05:57:23.155673400Z,ESU5,6363.75,49,6364,68,3/13/2026 5:57:23 AM
+60,2026-03-13T05:57:23.028898400Z,ESU5,6363.75,48,6364,68,3/13/2026 5:57:23 AM
+59,2026-03-13T05:57:22.887132700Z,ESU5,6363.75,46,6364,68,3/13/2026 5:57:22 AM
+58,2026-03-13T05:57:22.676916300Z,ESU5,6363.75,44,6364,68,3/13/2026 5:57:22 AM
+57,2026-03-13T05:57:22.520328000Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:22 AM
+56,2026-03-13T05:57:22.405244600Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:22 AM
+55,2026-03-13T05:57:22.294674100Z,ESU5,6363.75,41,6364,69,3/13/2026 5:57:22 AM
+54,2026-03-13T05:57:22.163467900Z,ESU5,6363.75,38,6364,69,3/13/2026 5:57:22 AM
+53,2026-03-13T05:57:21.992443000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
+52,2026-03-13T05:57:21.872338400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
+51,2026-03-13T05:57:21.742801800Z,ESU5,6363.75,37,6364,71,3/13/2026 5:57:21 AM
+50,2026-03-13T05:57:21.557711100Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
+49,2026-03-13T05:57:21.434977000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
+48,2026-03-13T05:57:21.285751600Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
+47,2026-03-13T05:57:21.184392000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
+46,2026-03-13T05:57:21.076843800Z,ESU5,6363.75,34,6364,73,3/13/2026 5:57:21 AM
+45,2026-03-13T05:57:20.894131800Z,ESU5,6363.75,32,6364,73,3/13/2026 5:57:20 AM
+44,2026-03-13T05:57:20.759550000Z,ESU5,6363.75,30,6364,73,3/13/2026 5:57:20 AM
+43,2026-03-13T05:57:20.641547900Z,ESU5,6363.75,28,6364,73,3/13/2026 5:57:20 AM
+42,2026-03-13T05:57:20.540293200Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
+41,2026-03-13T05:57:20.439034600Z,ESU5,6363.75,27,6364,74,3/13/2026 5:57:20 AM
+40,2026-03-13T05:57:20.337795000Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
+39,2026-03-13T05:57:20.236606600Z,ESU5,6363.75,27,6364,72,3/13/2026 5:57:20 AM
+38,2026-03-13T05:57:20.135417700Z,ESU5,6363.75,30,6364,72,3/13/2026 5:57:20 AM
+37,2026-03-13T05:57:20.034272700Z,ESU5,6363.75,32,6364,72,3/13/2026 5:57:20 AM
+36,2026-03-13T05:57:19.933096800Z,ESU5,6363.75,34,6364,72,3/13/2026 5:57:19 AM
+35,2026-03-13T05:57:19.832012600Z,ESU5,6363.75,36,6364,72,3/13/2026 5:57:19 AM
+34,2026-03-13T05:57:19.730878500Z,ESU5,6363.75,38,6364,72,3/13/2026 5:57:19 AM
+33,2026-03-13T05:57:19.629787400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
+32,2026-03-13T05:57:19.528697000Z,ESU5,6363.75,38,6364,70,3/13/2026 5:57:19 AM
+31,2026-03-13T05:57:19.427077000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
+30,2026-03-13T05:57:19.326450300Z,ESU5,6363.75,39,6364,71,3/13/2026 5:57:19 AM
+29,2026-03-13T05:57:19.225427900Z,ESU5,6363.75,39,6364,70,3/13/2026 5:57:19 AM
+28,2026-03-13T05:57:19.123339800Z,ESU5,6363.75,40,6364,70,3/13/2026 5:57:19 AM
+27,2026-03-13T05:57:19.021936500Z,ESU5,6363.75,41,6364,70,3/13/2026 5:57:19 AM
+26,2026-03-13T05:57:18.920859200Z,ESU5,6363.75,45,6364,70,3/13/2026 5:57:18 AM
+25,2026-03-13T05:57:18.820148400Z,ESU5,6363.75,46,6364,70,3/13/2026 5:57:18 AM
+24,2026-03-13T05:57:18.718988100Z,ESU5,6363.75,46,6364,69,3/13/2026 5:57:18 AM
+23,2026-03-13T05:57:18.618299700Z,ESU5,6363.75,49,6364,69,3/13/2026 5:57:18 AM
+22,2026-03-13T05:57:18.517138100Z,ESU5,6363.75,52,6364,69,3/13/2026 5:57:18 AM
+21,2026-03-13T05:57:18.416335200Z,ESU5,6363.75,53,6364,69,3/13/2026 5:57:18 AM
+20,2026-03-13T05:57:18.315335900Z,ESU5,6363.75,55,6364,69,3/13/2026 5:57:18 AM
+19,2026-03-13T05:57:18.214368600Z,ESU5,6363.75,57,6364,69,3/13/2026 5:57:18 AM
+18,2026-03-13T05:57:18.113754700Z,ESU5,6363.75,57,6364,65,3/13/2026 5:57:18 AM
+17,2026-03-13T05:57:18.012706500Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:18 AM
+16,2026-03-13T05:57:17.911672600Z,ESU5,6363.75,54,6364,65,3/13/2026 5:57:17 AM
+15,2026-03-13T05:57:17.810879900Z,ESU5,6363.75,52,6364,65,3/13/2026 5:57:17 AM
+14,2026-03-13T05:57:17.710089700Z,ESU5,6363.75,49,6364,65,3/13/2026 5:57:17 AM
+13,2026-03-13T05:57:17.609388300Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+12,2026-03-13T05:57:17.508697400Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+11,2026-03-13T05:57:17.408027000Z,ESU5,6363.75,53,6364,67,3/13/2026 5:57:17 AM
+10,2026-03-13T05:57:17.307406200Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
+9,2026-03-13T05:57:17.206714300Z,ESU5,6363.75,49,6364,71,3/13/2026 5:57:17 AM
+8,2026-03-13T05:57:17.105530300Z,ESU5,6363.75,48,6364,71,3/13/2026 5:57:17 AM
+7,2026-03-13T05:57:17.004796800Z,ESU5,6363.75,47,6364,71,3/13/2026 5:57:17 AM
+6,2026-03-13T05:57:16.904113900Z,ESU5,6363.75,46,6364,71,3/13/2026 5:57:16 AM
+5,2026-03-13T05:57:16.803381900Z,ESU5,6363.75,43,6364,71,3/13/2026 5:57:16 AM
+4,2026-03-13T05:57:16.701969600Z,ESU5,6363.75,43,6364,72,3/13/2026 5:57:16 AM
+3,2026-03-13T05:57:16.600765600Z,ESU5,6363.75,43,6364,73,3/13/2026 5:57:16 AM
+2,2026-03-13T05:57:16.499986300Z,ESU5,6363.75,43,6364,74,3/13/2026 5:57:16 AM
+1,2026-03-13T05:57:16.399241700Z,ESU5,6363.75,43,6364,75,3/13/2026 5:57:16 AM
+0,2026-03-13T05:57:16.264038000Z,ESU5,6363.75,43,6364,76,3/13/2026 5:57:16 AM

src/DataStreamSuite/BlueOrbDSL/__init__.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+import os
+import subprocess
+import threading
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class DataPacket:
+    state: List[Any]
+    deleted_ids: List[int]
+    added_ids: List[int]
+
+
+@dataclass
+class DslResponse:
+    data_packet: DataPacket | None
+    req_id: str | None
+    error: str | None
+
+
+def _resolve_interop_exe() -> str:
+    env_path = os.environ.get("BLUEORB_DSL_INTEROP")
+    if env_path and os.path.isfile(env_path):
+        return env_path
+    candidates = [
+        "/workspaces/spark/build/Debug/bin/net8.0/BlueOrbDSLInteropServer",
+        "/workspaces/spark/build/Release/bin/net8.0/BlueOrbDSLInteropServer",
+    ]
+    for path in candidates:
+        if os.path.isfile(path):
+            return path
+    raise FileNotFoundError(
+        "BlueOrbDSLInterop executable not found. "
+        "Set BLUEORB_DSL_INTEROP to the full path."
+    )
+
+_interop_proc: Optional[subprocess.Popen] = None
+_reader_thread: Optional[threading.Thread] = None
+_callbacks: Dict[str, Callable[[DslResponse], None]] = {}
+_first_response_events: Dict[str, threading.Event] = {}
+_first_responses: Dict[str, DslResponse] = {}
+_state_lock = threading.Lock()
+
+
+def start_interop() -> None:
+    """
+    Start the BlueOrbDSLInteropServer process if it's not already running.
+    """
+    global _interop_proc
+    if _interop_proc is not None and _interop_proc.poll() is None:
+        return
+    exe = _resolve_interop_exe()
+    _interop_proc = subprocess.Popen(
+        [exe],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    _start_reader_thread()
+
+
+def stop_interop() -> None:
+    """
+    Stop the BlueOrbDSLInteropServer process if it's running.
+    """
+    global _interop_proc
+    if _interop_proc is None:
+        return
+    if _interop_proc.poll() is None and _interop_proc.stdin is not None:
+        _interop_proc.stdin.write(":exit\n")
+        _interop_proc.stdin.flush()
+        _interop_proc.stdin.close()
+        _interop_proc.wait(timeout=5)
+    global _reader_thread
+    if _reader_thread is not None:
+        _reader_thread.join(timeout=1)
+        _reader_thread = None
+    with _state_lock:
+        _callbacks.clear()
+        _first_response_events.clear()
+        _first_responses.clear()
+    _interop_proc = None
+
+
+def get_interop_status() -> tuple[bool, int | None]:
+    """
+    Returns current interop process status as (is_running, exit_code).
+    exit_code is None when process was never started or is still running.
+    """
+    proc = _interop_proc
+    if proc is None:
+        return (False, None)
+    rc = proc.poll()
+    return (rc is None, rc)
+
+
+def get_dsl(dsl: str, req_id: str, callback: Callable[[DslResponse], None]) -> DslResponse:
+    """
+    Sends the DSL to BlueOrbDSLInteropServer and returns the first response for this req_id.
+    Subsequent streaming updates for the same req_id are delivered via callback.
+    Requires start_interop() to be called first.
+    """
+    assert _interop_proc is not None and _interop_proc.poll() is None, (
+        "Interop process is not running. Call start_interop() first."
+    )
+    assert _interop_proc.stdin is not None
+
+    payload = {"dsl": dsl, "req_id": req_id}
+    first_event = threading.Event()
+    with _state_lock:
+        _callbacks[req_id] = callback
+        _first_response_events[req_id] = first_event
+        _first_responses.pop(req_id, None)
+
+    _interop_proc.stdin.write(json.dumps(payload) + "\n")
+    _interop_proc.stdin.flush()
+
+    if not first_event.wait(timeout=10):
+        with _state_lock:
+            _first_response_events.pop(req_id, None)
+            _callbacks.pop(req_id, None)
+        raise TimeoutError(f"No response received for req_id '{req_id}' from BlueOrbDSLInterop.")
+
+    with _state_lock:
+        response = _first_responses.pop(req_id, None)
+        _first_response_events.pop(req_id, None)
+    if response is None:
+        raise RuntimeError(f"Missing first response for req_id '{req_id}'.")
+    return response
+
+
+def _start_reader_thread() -> None:
+    global _reader_thread
+    if _reader_thread is not None and _reader_thread.is_alive():
+        return
+    _reader_thread = threading.Thread(target=_reader_loop, daemon=True)
+    _reader_thread.start()
+
+
+def _reader_loop() -> None:
+    while True:
+        proc = _interop_proc
+        if proc is None or proc.poll() is not None:
+            return
+        assert proc.stdout is not None
+        raw = proc.stdout.readline()
+        if raw == "":
+            return
+
+        line = raw.strip()
+        if not line:
+            continue
+
+        try:
+            payload = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        response = DslResponse(
+            data_packet=DataPacket(state=[payload.get("data_packet")], deleted_ids=[], added_ids=[]),
+            req_id=payload.get("req_id"),
+            error=payload.get("error"),
+        )
+        req_id = response.req_id
+        if not req_id:
+            continue
+
+        callback: Optional[Callable[[DslResponse], None]]
+        event: Optional[threading.Event]
+        with _state_lock:
+            callback = _callbacks.get(req_id)
+            event = _first_response_events.get(req_id)
+            if event is not None and req_id not in _first_responses:
+                _first_responses[req_id] = response
+                event.set()
+
+        if callback is not None:
+            try:
+                callback(response)
+            except Exception:
+                # Keep the reader alive if callback logic fails.
+                pass

src/DataStreamSuite/BlueOrbDSL/run_blueorb_dsl.py

@@ -0,0 +1,33 @@
+import os
+import sys
+import uuid
+
+_here = os.path.dirname(os.path.abspath(__file__))
+_parent = os.path.dirname(_here)
+if _parent not in sys.path:
+    sys.path.insert(0, _parent)
+
+from BlueOrbDSL import get_dsl, start_interop, stop_interop, DslResponse
+
+
+def on_packet(response: DslResponse) -> None:
+    print("callback from DSL Server, DslResponse:", response)
+
+
+def main() -> None:
+    start_interop()
+    try:
+        while True:
+            dsl = input("Enter DSL (or 'quit'): ").strip()
+            if dsl.lower() == "quit":
+                break
+            if not dsl:
+                continue
+            response = get_dsl(dsl, str(uuid.uuid4()), on_packet)
+            print("return:", response)
+    finally:
+        stop_interop()
+
+
+if __name__ == "__main__":
+    main()

src/DataStreamSuite/BlueOrbDSL/run_dsqa.py

@@ -0,0 +1,844 @@
+from __future__ import annotations
+
+import csv
+import time
+import os
+import sys
+import threading
+import uuid
+from collections import deque
+from datetime import datetime
+from typing import Any
+
+from rich.text import Text
+from rich.style import Style
+from textual import events
+from textual.app import App, ComposeResult
+from textual.binding import Binding
+from textual.containers import Container, Horizontal, Vertical
+from textual.widgets.text_area import TextAreaTheme
+from textual.widgets import Button, DataTable, Input, Static, TabbedContent, TabPane, TextArea, Tree
+
+_here = os.path.dirname(os.path.abspath(__file__))
+_parent = os.path.dirname(_here)
+if _parent not in sys.path:
+    sys.path.insert(0, _parent)
+
+from BlueOrbDSL import DslResponse, get_dsl, get_interop_status, start_interop, stop_interop
+
+
+class DSQALiveConsole(App):
+    DEFAULT_QUERY = (
+        'FROM market.ticks\n'
+        'WHERE symbol IN ("BTCUSD", "ETHUSD")\n'
+        "SELECT symbol, price, change, time"
+    )
+    LAST_QUERY_PATH = os.path.join(_here, "DSQA_LAST_EXECUTED_QUERY.txt")
+    DSL_HIGHLIGHT_QUERY = r"""
+    ; comments
+    (comment) @comment
+
+    ; assignment vars: lhs in x = SomeStream(...)
+    (assignment_expression
+      left: (identifier) @variable)
+
+    ; stream operators (function names ending with Stream)
+    (call_expression
+      function: (identifier) @function
+      (#match? @function "^[A-Za-z_][A-Za-z0-9_]*Stream$"))
+
+    ; normal function calls
+    (call_expression
+      function: (identifier) @function.method)
+
+    ; arguments and literals
+    (arguments (identifier) @parameter)
+    (string) @string
+    (number) @number
+    (true) @constant.builtin
+    (false) @constant.builtin
+    (null) @constant.builtin
+    """
+
+    BINDINGS = [
+        Binding("ctrl+r", "execute", "Execute"),
+        Binding("f11", "execute", show=False),
+        Binding("alt+enter", "execute", show=False),
+        # Override Textual's default Ctrl+Q quit to avoid accidental VS Code close.
+        Binding("ctrl+q", "noop", show=False),
+        Binding("ctrl+x", "quit", "Quit"),
+    ]
+    CSS = """
+    Screen {
+        background: #060a1c;
+        color: #d8dff4;
+    }
+
+    #window_close_button {
+        width: 3;
+        min-width: 3;
+        height: 1;
+        margin: 0;
+        border: none;
+        background: #3a1015;
+        color: #ff9ca8;
+        text-style: bold;
+    }
+
+    #window_close_button:hover,
+    #window_close_button:focus {
+        background: #5a1620;
+        color: #ffd6db;
+    }
+
+    #root {
+        padding: 0 1 0 1;
+        height: 1fr;
+    }
+
+    #query_panel {
+        background: #070d27;
+        height: 33%;
+        margin-top: 1;
+    }
+
+    #query_header_row {
+        height: 1;
+        border-bottom: solid #596285;
+    }
+
+    #query_header {
+        width: auto;
+        color: #9eb0d4;
+        content-align: left middle;
+    }
+
+    #query_header_spacer {
+        width: 1fr;
+    }
+
+    #query_input {
+        border: solid #697597;
+        background: #050b22;
+        color: #cfd7eb;
+        height: 1fr;
+        padding-left: 1;
+    }
+
+    #results_panel {
+        background: #070d27;
+        height: 1fr;
+        margin-top: 1;
+    }
+
+    #results_toolbar {
+        height: 3;
+        padding: 0 1;
+    }
+
+    #results_left {
+        width: auto;
+        height: 3;
+        margin-top: 0;
+    }
+
+    #stream_toggle_button {
+        width: auto;
+        min-width: 11;
+        margin-top: 0;
+        margin-left: 1;
+        border: none;
+        background: #070d27;
+        color: #79afe7;
+    }
+
+    #stream_toggle_button:hover,
+    #stream_toggle_button:focus {
+        background: #111a3f;
+        color: #9bc9f6;
+    }
+
+    #execute_button {
+        width: 5;
+        min-width: 5;
+        margin-top: 0;
+        margin-left: 1;
+        border: none;
+        background: #10341f;
+        color: #7ef1ac;
+        text-style: bold;
+    }
+
+    #execute_button:hover,
+    #execute_button:focus {
+        background: #1a4f30;
+        color: #b3ffd0;
+    }
+
+    #results_actions {
+        width: auto;
+        height: 3;
+        margin-top: 0;
+    }
+
+    #results_spacer {
+        width: 1fr;
+    }
+
+    #results_search {
+        width: 30;
+        min-width: 24;
+        margin-right: 1;
+        border: none;
+        background: #060d28;
+        color: #d5def3;
+        padding: 0 1;
+    }
+
+    #search_button,
+    #copy_button,
+    #export_button {
+        width: auto;
+        min-width: 11;
+        margin-left: 1;
+        border: none;
+        background: #070d27;
+        color: #79afe7;
+    }
+
+    #search_button:hover,
+    #copy_button:hover,
+    #export_button:hover,
+    #search_button:focus,
+    #copy_button:focus,
+    #export_button:focus {
+        background: #111a3f;
+        color: #9bc9f6;
+    }
+
+    #results_table {
+        border: solid #697597;
+        background: #050b22;
+        height: 1fr;
+    }
+
+    #results_table .datatable--header,
+    #results_table .datatable--header-hover,
+    #results_table .datatable--header-cursor {
+        background: #0c1535;
+        color: #6ea6df;
+        text-style: bold;
+    }
+
+    #results_table .datatable--even-row {
+        background: #050b22;
+    }
+
+    #results_table .datatable--odd-row {
+        background: #07102a;
+    }
+
+    #results_table .datatable--cursor {
+        background: #19335b;
+        color: #dbe8ff;
+    }
+
+    #results_table .datatable--fixed-cursor {
+        background: #132949;
+        color: #c8dbff;
+    }
+
+    #results_table.error .datatable--odd-row,
+    #results_table.error .datatable--even-row,
+    #results_table.error .datatable--cursor,
+    #results_table.error .datatable--fixed-cursor {
+        color: #f17486;
+        text-style: bold;
+    }
+
+    #results_panel.paused {
+        background: #150f1f;
+    }
+
+    #results_table.paused {
+        background: #130f18;
+    }
+
+    #results_table.paused .datatable--header,
+    #results_table.paused .datatable--header-hover,
+    #results_table.paused .datatable--header-cursor {
+        background: #262338;
+        color: #8b90a1;
+        text-style: bold;
+    }
+
+    #results_tabs {
+        height: 1fr;
+        background: #070d27;
+    }
+
+    #data_tab,
+    #tree_tab {
+        padding: 0;
+    }
+
+    #ops_tree {
+        border: solid #697597;
+        background: #050b22;
+        color: #cfd7eb;
+        height: 1fr;
+    }
+
+    #bottom_status_bar {
+        border-top: solid #596285;
+        background: #070d27;
+        height: 2;
+        margin: 0;
+        padding: 0 1;
+    }
+
+    #status_right_wrap {
+        width: 1fr;
+    }
+
+    #status_row {
+        height: 1;
+        margin-top: 0;
+        width: 1fr;
+    }
+
+    #live_status {
+        width: 1fr;
+        color: #b3bed9;
+        content-align: right middle;
+    }
+    """
+
+    def compose(self) -> ComposeResult:
+        with Container(id="root"):
+            with Vertical(id="query_panel"):
+                with Horizontal(id="query_header_row"):
+                    yield Static("DSL Query", id="query_header")
+                    yield Static("", id="query_header_spacer")
+                    yield Button("✕", id="window_close_button")
+                yield TextArea(
+                    text=self._load_last_query(),
+                    id="query_input",
+                    language="javascript",
+                    theme="vscode_dark",
+                    show_line_numbers=True,
+                )
+            with Vertical(id="results_panel"):
+                with Horizontal(id="results_toolbar"):
+                    with Horizontal(id="results_left"):
+                        yield Button("▶", id="execute_button")
+                        yield Button("⏸ Pause", id="stream_toggle_button")
+                    yield Static("", id="results_spacer")
+                    with Horizontal(id="results_actions"):
+                        yield Input(placeholder="Search: col:value", id="results_search")
+                        yield Button("⌕ Search", id="search_button")
+                        yield Button("⧉ Copy", id="copy_button")
+                        yield Button("⇩ Export", id="export_button")
+                with TabbedContent(initial="data_tab", id="results_tabs"):
+                    with TabPane("Data", id="data_tab"):
+                        table = DataTable(id="results_table")
+                        table.cursor_type = "cell"
+                        table.show_cursor = True
+                        table.zebra_stripes = True
+                        yield table
+                    with TabPane("Operators", id="tree_tab"):
+                        tree = Tree("Assemblies", id="ops_tree")
+                        tree.show_root = False
+                        yield tree
+        with Horizontal(id="bottom_status_bar"):
+            with Horizontal(id="status_right_wrap"):
+                with Horizontal(id="status_row"):
+                    yield Static(
+                        "Rows: 0 | Rate: 0.0/s | Total Updates: 0 | Uptime: 00:00:00 | Last Update: --:--:--",
+                        id="live_status",
+                    )
+
+    def on_mount(self) -> None:
+        self._update_timestamps: deque[float] = deque()
+        self._last_update_at: datetime | None = None
+        self._stream_started_at: float | None = None
+        self._total_updates = 0
+        self._last_row_count = 0
+        self._current_col_names: list[str] = []
+        self._current_rows: list[dict[str, Any]] = []
+        self._column_width_cache: dict[str, int] = {}
+        self._ops_catalog_rows: list[dict[str, Any]] = []
+        self._ops_catalog_error: str | None = None
+        self._ops_catalog_loaded = False
+        self._ops_catalog_loading = False
+        self._interop_is_running = False
+        self._interop_exit_code: int | None = None
+        self._grid_updates_enabled = True
+        self._paused_latest_response: DslResponse | None = None
+        self._ensure_interop_started()
+        self._refresh_ops_tree()
+        self._refresh_ops_tree_async()
+        self._configure_query_highlighting()
+        self._set_pause_visuals()
+        self.set_interval(0.5, self._refresh_status)
+
+    def _configure_query_highlighting(self) -> None:
+        query_input = self.query_one("#query_input", TextArea)
+        query_input.language = "javascript"
+        dsl_theme = TextAreaTheme(
+            "dsqa_ide",
+            base_style=Style.parse("#d7deee on #171a22"),
+            gutter_style=Style.parse("#8a93a8 on #1c212b"),
+            cursor_style=Style.parse("#ffffff on #2b3242"),
+            cursor_line_style=Style.parse("on #202534"),
+            cursor_line_gutter_style=Style.parse("#a6afc4 on #202534"),
+            selection_style=Style.parse("on #2a3f67"),
+            bracket_matching_style=Style.parse("bold #7fd9ff"),
+            syntax_styles={
+                "comment": Style.parse("italic #6f778a"),
+                "variable": Style.parse("bold #4ec9b0"),
+                "function": Style.parse("bold #4fc1ff"),
+                "function.method": Style.parse("bold #70d4ff"),
+                "parameter": Style.parse("#c6d4f8"),
+                "string": Style.parse("#d7ba7d"),
+                "number": Style.parse("#b5cea8"),
+                "constant.builtin": Style.parse("bold #569cd6"),
+            },
+        )
+        query_input.register_theme(dsl_theme)
+        query_input.theme = "dsqa_ide"
+        try:
+            query_input.update_highlight_query(self.DSL_HIGHLIGHT_QUERY)
+        except Exception:
+            pass
+
+    def on_shutdown(self) -> None:
+        stop_interop()
+
+    @staticmethod
+    def _ensure_interop_started() -> None:
+        try:
+            start_interop()
+        except Exception:
+            pass
+
+    def action_execute(self) -> None:
+        tabs = self.query_one("#results_tabs", TabbedContent)
+        tabs.active = "data_tab"
+        self._grid_updates_enabled = True
+        self._set_stream_toggle_label()
+        self._set_pause_visuals()
+        self._execute_query_async()
+
+    def action_noop(self) -> None:
+        pass
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        button_id = event.button.id
+        if button_id == "search_button":
+            self._search_results()
+        elif button_id == "copy_button":
+            self._copy_selected_cell()
+        elif button_id == "export_button":
+            self._export_results_csv()
+        elif button_id == "execute_button":
+            self.action_execute()
+        elif button_id == "window_close_button":
+            self.exit()
+        elif button_id == "stream_toggle_button":
+            self._toggle_stream_painting()
+
+    def on_input_submitted(self, event: Input.Submitted) -> None:
+        if event.input.id == "results_search":
+            self._search_results()
+
+    def on_click(self, event: events.Click) -> None:
+        if getattr(event.widget, "id", None) != "ops_tree":
+            return
+        line = event.style.meta.get("line")
+        if line is None:
+            return
+        tree = self.query_one("#ops_tree", Tree)
+        node = tree.get_node_at_line(line)
+        if node is None:
+            return
+        node_data = node.data
+        if not isinstance(node_data, dict) or node_data.get("kind") != "constructor":
+            return
+        self._append_operator_to_query(self._stringify(node_data.get("op", "")).strip())
+
+    def _append_operator_to_query(self, op_name: str) -> None:
+        if not op_name:
+            return
+        query_input = self.query_one("#query_input", TextArea)
+        existing = query_input.text
+        if existing.strip():
+            separator = "" if existing.endswith("\n") else "\n"
+            query_input.text = f"{existing}{separator}{op_name}"
+        else:
+            query_input.text = op_name
+        query_input.focus()
+
+    def _execute_query_async(self) -> None:
+        self._ensure_interop_started()
+        query = self.query_one(TextArea).text.strip()
+        if not query:
+            return
+        self._column_width_cache.clear()
+        self._save_last_query(query)
+        self._update_timestamps.clear()
+        self._last_update_at = None
+        self._stream_started_at = time.monotonic()
+        self._total_updates = 0
+        self._last_row_count = 0
+        self._paused_latest_response = None
+        self._refresh_status()
+
+        req_id = str(uuid.uuid4())
+
+        def _run() -> None:
+            try:
+                def _on_response(resp: DslResponse) -> None:
+                    self.call_from_thread(self._render_response, resp)
+
+                get_dsl(query, req_id, _on_response)
+            except Exception as exc:
+                error_resp = DslResponse(data_packet=None, req_id=req_id, error=str(exc))
+                self.call_from_thread(self._render_response, error_resp)
+
+        threading.Thread(target=_run, daemon=True).start()
+
+    def _render_response(self, response: DslResponse) -> None:
+        if not self._grid_updates_enabled:
+            self._paused_latest_response = response
+            return
+        self._render_response_internal(response)
+
+    def _render_response_internal(self, response: DslResponse) -> None:
+        self._record_update()
+        table = self.query_one(DataTable)
+        table.remove_class("error")
+        prev_scroll_x = table.scroll_offset.x
+        prev_scroll_y = table.scroll_offset.y
+        table.clear(columns=True)
+
+        if response.error:
+            table.add_class("error")
+            error_text = self._stringify(response.error)
+            error_lines = error_text.splitlines() or [error_text]
+            self._current_col_names = []
+            self._current_rows = []
+            self._last_row_count = len(error_lines)
+            self._refresh_ops_tree()
+            table.add_columns("error")
+            for line in error_lines:
+                table.add_row(Text(line, style="bold #f17486"))
+            return
+
+        if response.data_packet is None or not response.data_packet.state:
+            self._current_col_names = []
+            self._current_rows = []
+            self._last_row_count = 0
+            self._refresh_ops_tree()
+            table.add_columns("empty")
+            table.add_row("no data")
+            return
+
+        data_packet = response.data_packet.state[0]
+        columns = data_packet.get("columns", [])
+        rows = data_packet.get("all", [])
+        if not isinstance(rows, list):
+            rows = []
+
+        col_names = [c.get("name", "") for c in columns]
+        if not col_names:
+            col_names = list(rows[0].keys()) if rows else ["empty"]
+
+        self._current_col_names = col_names
+        self._current_rows = rows
+        self._last_row_count = len(rows)
+        self._refresh_ops_tree()
+        for col_name in col_names:
+            width = self._column_width_cache.get(col_name, len(col_name))
+            for row in rows:
+                if not isinstance(row, dict):
+                    continue
+                cell_text = self._stringify(row.get(col_name, ""))
+                width = max(width, len(cell_text))
+            self._column_width_cache[col_name] = width
+            table.add_column(col_name, width=width)
+        for row in rows:
+            table.add_row(*[self._format_cell(c, row.get(c, "")) for c in col_names])
+        # Preserve user's viewport during streaming refreshes.
+        table.scroll_to(
+            x=min(prev_scroll_x, table.max_scroll_x),
+            y=min(prev_scroll_y, table.max_scroll_y),
+            animate=False,
+        )
+
+    @staticmethod
+    def _stringify(value: Any) -> str:
+        return "" if value is None else str(value)
+
+    @staticmethod
+    def _format_cell(column_name: str, value: Any) -> str | Text:
+        text = DSQALiveConsole._stringify(value)
+        lowered = column_name.lower()
+        if lowered == "id":
+            return Text(text, style="bold #9eb1d8")
+        if lowered == "symbol":
+            return Text(text, style="bold #dce6ff")
+        if lowered == "price":
+            return Text(text, style="bold #d4ddf2")
+        if lowered == "time":
+            return Text(text, style="#aab5d0")
+        if lowered == "change":
+            cleaned = text.strip()
+            if cleaned.startswith("-"):
+                return Text(text, style="bold #f17486")
+            if cleaned.startswith("+"):
+                return Text(text, style="bold #92d67e")
+            return Text(text, style="bold #c7d3ef")
+        if lowered == "bid":
+            return Text(text, style="bold #8cf3c0 on #123526")
+        if lowered == "ask":
+            return Text(text, style="bold #ffd5d9 on #3b1b25")
+        return text
+
+    def _record_update(self) -> None:
+        now = time.monotonic()
+        self._update_timestamps.append(now)
+        self._total_updates += 1
+        if self._stream_started_at is None:
+            self._stream_started_at = now
+        self._last_update_at = datetime.now()
+        self._refresh_status()
+
+    def _refresh_status(self) -> None:
+        self._refresh_interop_heartbeat()
+        now = time.monotonic()
+        cutoff = now - 1.0
+        while self._update_timestamps and self._update_timestamps[0] < cutoff:
+            self._update_timestamps.popleft()
+
+        rate = float(len(self._update_timestamps))
+        if self._stream_started_at is not None:
+            uptime_seconds = max(0, int(now - self._stream_started_at))
+            uptime = self._format_duration(uptime_seconds)
+        else:
+            uptime = "--:--:--"
+        last_update = self._last_update_at.strftime("%H:%M:%S") if self._last_update_at else "--:--:--"
+        status = Text()
+        if self._interop_is_running:
+            status.append("Interop: ● UP", style="bold #92d67e")
+        elif self._interop_exit_code is not None:
+            status.append(f"Interop: ● DOWN({self._interop_exit_code})", style="bold #f17486")
+        else:
+            status.append("Interop: ● DOWN", style="bold #8b90a1")
+        status.append(" | ")
+        status.append(f"Rows: {self._last_row_count} | Rate: {rate:.1f}/s | ")
+        status.append(f"Total Updates: {self._total_updates}", style="bold #79afe7")
+        status.append(f" | Uptime: {uptime} | ")
+        status.append(f"Last Update: {last_update}", style="bold #79afe7")
+        self.query_one("#live_status", Static).update(status)
+
+    def _refresh_interop_heartbeat(self) -> None:
+        is_running, exit_code = get_interop_status()
+        self._interop_is_running = is_running
+        self._interop_exit_code = exit_code
+
+    @staticmethod
+    def _format_duration(total_seconds: int) -> str:
+        hours = total_seconds // 3600
+        minutes = (total_seconds % 3600) // 60
+        seconds = total_seconds % 60
+        return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+    def _search_results(self) -> None:
+        raw = self.query_one("#results_search", Input).value.strip()
+        if ":" not in raw:
+            return
+        col_part, value_part = raw.split(":", 1)
+        target_col = col_part.strip()
+        target_value = value_part.strip()
+        if not target_col:
+            return
+
+        lookup = {name.lower(): idx for idx, name in enumerate(self._current_col_names)}
+        col_idx = lookup.get(target_col.lower())
+        if col_idx is None:
+            return
+
+        col_name = self._current_col_names[col_idx]
+        needle = target_value.lower()
+        for row_idx, row in enumerate(self._current_rows):
+            if not isinstance(row, dict):
+                continue
+            cell_value = self._stringify(row.get(col_name, ""))
+            if needle in cell_value.lower():
+                table = self.query_one("#results_table", DataTable)
+                table.move_cursor(row=row_idx, column=col_idx)
+                table.focus()
+                return
+
+    def _copy_selected_cell(self) -> None:
+        table = self.query_one("#results_table", DataTable)
+        if table.row_count == 0 or not self._current_col_names:
+            return
+        coordinate = table.cursor_coordinate
+        try:
+            cell = table.get_cell_at(coordinate)
+        except Exception:
+            return
+        if isinstance(cell, Text):
+            value = cell.plain
+        else:
+            value = self._stringify(cell)
+        self.copy_to_clipboard(value)
+
+    def _export_results_csv(self) -> None:
+        if not self._current_col_names:
+            return
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        export_path = os.path.join(_here, f"DSQA_RESULTS_EXPORT_{timestamp}.csv")
+        with open(export_path, "w", newline="", encoding="utf-8") as csv_file:
+            writer = csv.writer(csv_file)
+            writer.writerow(self._current_col_names)
+            for row in self._current_rows:
+                if isinstance(row, dict):
+                    writer.writerow([self._stringify(row.get(col, "")) for col in self._current_col_names])
+
+    def _refresh_ops_tree(self) -> None:
+        tree = self.query_one("#ops_tree", Tree)
+        root = tree.root
+        root.remove_children()
+
+        if self._ops_catalog_error:
+            root.add(Text(self._ops_catalog_error, style="bold #f17486"))
+            root.expand()
+            return
+
+        grouped: dict[str, dict[str, dict[str, set[str]]]] = {}
+        for row in self._ops_catalog_rows:
+            if not isinstance(row, dict):
+                continue
+            row_lookup = {self._stringify(k).lower(): v for k, v in row.items()}
+            assembly_name = self._stringify(row_lookup.get("assembly_name", "")).strip() or "(unknown assembly)"
+            namespace = self._stringify(row_lookup.get("namespace", "")).strip() or "(global)"
+            op = self._stringify(row_lookup.get("op", "")).strip() or "(unknown op)"
+            constructor = self._stringify(row_lookup.get("constructor", "")).strip() or "(none)"
+            grouped.setdefault(assembly_name, {}).setdefault(namespace, {}).setdefault(op, set()).add(constructor)
+
+        if not grouped:
+            root.add(Text("No operator metadata available", style="#8ea0c6"))
+            root.expand()
+            return
+
+        for assembly_name in sorted(grouped.keys(), key=str.lower):
+            assembly_node = root.add(Text(assembly_name, style="bold #79afe7"))
+            for namespace in sorted(grouped[assembly_name].keys(), key=str.lower):
+                namespace_node = assembly_node.add(Text(namespace, style="#cfd7eb"))
+                for op in sorted(grouped[assembly_name][namespace].keys(), key=str.lower):
+                    op_node = namespace_node.add(
+                        Text(op, style="#b8c8e7"),
+                        data={"kind": "op", "op": op},
+                    )
+                    for constructor in sorted(grouped[assembly_name][namespace][op], key=str.lower):
+                        op_node.add(
+                            Text(constructor, style="#9eb0d4"),
+                            data={"kind": "constructor", "op": op, "constructor": constructor},
+                        )
+                    op_node.expand()
+                namespace_node.expand()
+            assembly_node.expand()
+        root.expand()
+
+    def _refresh_ops_tree_async(self) -> None:
+        if self._ops_catalog_loaded or self._ops_catalog_loading:
+            return
+        self._ensure_interop_started()
+        self._ops_catalog_loading = True
+        req_id = str(uuid.uuid4())
+
+        def _run() -> None:
+            try:
+                def _on_response(resp: DslResponse) -> None:
+                    self.call_from_thread(self._update_ops_catalog, resp)
+
+                get_dsl("OpsStream()", req_id, _on_response)
+            except Exception as exc:
+                error_resp = DslResponse(data_packet=None, req_id=req_id, error=str(exc))
+                self.call_from_thread(self._update_ops_catalog, error_resp)
+
+        threading.Thread(target=_run, daemon=True).start()
+
+    def _update_ops_catalog(self, response: DslResponse) -> None:
+        self._ops_catalog_loading = False
+        if response.error:
+            self._ops_catalog_rows = []
+            self._ops_catalog_error = self._stringify(response.error)
+            self._refresh_ops_tree()
+            return
+
+        if response.data_packet is None or not response.data_packet.state:
+            self._ops_catalog_rows = []
+            self._ops_catalog_error = "OpsStream() returned no data"
+            self._refresh_ops_tree()
+            return
+
+        data_packet = response.data_packet.state[0]
+        rows = data_packet.get("all", [])
+        if not isinstance(rows, list):
+            rows = []
+        normalized_rows: list[dict[str, Any]] = []
+        for row in rows:
+            if isinstance(row, dict):
+                normalized_rows.append(row)
+        self._ops_catalog_rows = normalized_rows
+        self._ops_catalog_error = None
+        self._ops_catalog_loaded = True
+        self._refresh_ops_tree()
+
+    def _toggle_stream_painting(self) -> None:
+        self._grid_updates_enabled = not self._grid_updates_enabled
+        self._set_stream_toggle_label()
+        self._set_pause_visuals()
+        if self._grid_updates_enabled and self._paused_latest_response is not None:
+            latest = self._paused_latest_response
+            self._paused_latest_response = None
+            self._render_response_internal(latest)
+
+    def _set_stream_toggle_label(self) -> None:
+        button = self.query_one("#stream_toggle_button", Button)
+        button.label = "⏸ Pause" if self._grid_updates_enabled else "▶ Play"
+
+    def _set_pause_visuals(self) -> None:
+        table = self.query_one("#results_table", DataTable)
+        panel = self.query_one("#results_panel", Vertical)
+        if self._grid_updates_enabled:
+            table.remove_class("paused")
+            panel.remove_class("paused")
+        else:
+            table.add_class("paused")
+            panel.add_class("paused")
+
+    def _load_last_query(self) -> str:
+        try:
+            with open(self.LAST_QUERY_PATH, "r", encoding="utf-8") as file:
+                text = file.read().strip()
+                return text if text else self.DEFAULT_QUERY
+        except FileNotFoundError:
+            return self.DEFAULT_QUERY
+        except OSError:
+            return self.DEFAULT_QUERY
+
+    def _save_last_query(self, query: str) -> None:
+        try:
+            with open(self.LAST_QUERY_PATH, "w", encoding="utf-8") as file:
+                file.write(query)
+        except OSError:
+            pass
+
+
+if __name__ == "__main__":
+    DSQALiveConsole().run()

src/DataStreamSuite/BlueOrbDSLInteropServer/BlueOrbDSLInteropServer.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AssemblyName>BlueOrbDSLInteropServer</AssemblyName>
+    <RootNamespace>BlueOrbDSLInteropServer</RootNamespace>
+  </PropertyGroup>
+
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\..\cs\Bts.Spark.Api\Bts.Spark.Api.csproj" />
+        <ProjectReference Include="..\DataStream\DataStream.csproj" />
+        <ProjectReference Include="..\PubSub\PubSub.csproj" />
+        <ProjectReference Include="..\BlueOrbOperatorStreams\BlueOrbOperatorStreams.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="Bts.Spark.Api" Version="$(SparkVersion)" />
+        <PackageReference Include="DataStream" Version="$(SparkVersion)" />
+        <PackageReference Include="PubSub" Version="$(SparkVersion)" />
+        <PackageReference Include="BlueOrbOperatorStreams" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
+
+</Project>

src/DataStreamSuite/BlueOrbDSLInteropServer/Program.cs

@@ -0,0 +1,122 @@
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+using BCG;
+
+using bts.BlueOrb;
+
+using Bts.DataStream;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+using Bts.PubSub;
+
+using MathNet.Numerics.Statistics;
+
+
+namespace BlueOrbDSLInteropServer {
+   // BlueOrbDSLInteropServer reads DSL lines from stdin, validates them,
+   // and writes one DataPacket JSON line to stdout per input.
+   internal static class Program {
+      private const string ExitSignal = ":exit";
+      private static SyncQueue<DslRequest> _work_queue = new SyncQueue<DslRequest>();
+      private static JsonSerializerOptions response_json_options = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      private static string _current_req_id;
+
+      // Request from Python
+      private sealed class DslRequest {
+         public string? dsl { get; set; }
+         public string? req_id { get; set; }
+      }
+
+      // Response envelope for Python: includes DataPacket, request id, and optional error.
+      private sealed class DslResponse {
+         public DataPacket? data_packet { get; set; }
+         public string? req_id { get; set; }
+         public string? error { get; set; }
+      }
+
+      private static void Main(string[] args) {
+         //System.Threading.Thread.Sleep(20000);
+         System.Threading.Thread thread = new Thread(_worker_loop);
+         thread.IsBackground = true;
+         thread.Start();
+         response_json_options.Converters.Add(new DataPacketJsonConverter {
+            IncludeCurrent = true,
+            UseObjectRows = true,
+         });
+         string? line;
+         while ((line = Console.ReadLine()) != null) {
+            line = line.Trim();
+            if (line.Length == 0) {
+               continue;
+            }
+            if (string.Equals(line, ExitSignal, StringComparison.OrdinalIgnoreCase)) {
+               break;
+            }
+
+            try {
+               DslRequest req = JsonSerializer.Deserialize<DslRequest>(line);
+               _work_queue.enqueue(req);
+            } catch (Exception ex) {
+               bts.Log.assert(false, ex.ToString());
+            }
+         }
+      }
+
+      private static void _worker_loop() {
+         Dictionary<string, ViewStream> req_map = new Dictionary<string, ViewStream>();
+         while (true) {
+            System.Threading.Thread.Sleep(100);
+            ViewStreamDatabase.instance.doWork();
+            var new_reqs = _work_queue.dequeue_all();
+            if (new_reqs == null || new_reqs.Count == 0) {
+               continue;
+            }
+            // ubsubscribing to previous stream(s)
+            foreach (var kv in req_map) {
+               kv.Value.onChange -= dp_obj => _on_dp(kv.Key, dp_obj);
+            }
+            foreach (var r in new_reqs) {
+               var req_id = string.IsNullOrWhiteSpace(r.req_id) ? Guid.NewGuid().ToString() : r.req_id;
+               _current_req_id = req_id;
+               try {
+                  DSLStream dsl_str = new DSLStream(r.dsl);
+                  req_map[req_id] = dsl_str;
+                  dsl_str.onChange += dp_obj => _on_dp(req_id, dp_obj);
+               } catch (Exception ex) {
+                  _write_error(req_id, ex.StackTrace);
+               }
+            }
+         }
+      }
+
+      private static void _on_dp(string req_id, object dp_obj) {
+         // eventhough we unsubscribe to all the other streams except the currrent one, 
+         // sometimes the onLastSub is not implemented and we still get updates, this masks them 
+         if (req_id != _current_req_id)
+            return;
+         DataPacket dp = dp_obj as DataPacket;
+         DslResponse response = new DslResponse() {
+            data_packet = dp,
+            req_id = req_id,
+            error = ""
+         };
+         var json = JsonSerializer.Serialize(response, response_json_options);
+         Console.WriteLine(json);
+      }
+
+      private static void _write_error(string req_id, string error) {
+         DslResponse response = new DslResponse() {
+            data_packet = null,
+            req_id = req_id,
+            error = error
+         };
+         var json = JsonSerializer.Serialize(response, response_json_options);
+         Console.WriteLine(json);
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/BarBuilderStream_depleted.cs

@@ -0,0 +1,514 @@
+// namespace BlueOrb.Operators.Signal;
+
+// using BCG;
+
+// using bts;
+
+// using System;
+// using System.Collections.Generic;
+// using System.Globalization;
+// using System.Linq;
+
+// [DataStreamUserRespresentation("BarBuilderStream")]
+// public class BarBuilderStream : ViewStream
+// {
+//    enum BarType
+//    {
+//       TIME,
+//       TICK,
+//       VOLUME,
+//       DOLLAR,
+//    }
+
+//    enum PriceSource
+//    {
+//       trade_price,
+//       mid,
+//       bid,
+//       ask,
+//    }
+
+//    enum VolumeSource
+//    {
+//       trade_size,
+//       none,
+//    }
+
+//    enum SessionReset
+//    {
+//       NONE,
+//       DAY,
+//       RTH,
+//       ETH,
+//    }
+
+//    sealed class BarState
+//    {
+//       public string Contract = string.Empty;
+//       public DateTime StartTs;
+//       public DateTime EndTs;
+//       public double Open;
+//       public double High;
+//       public double Low;
+//       public double Close;
+//       public double Volume;
+//       public int TradeCount;
+//       public double SumPriceSize;
+//       public double SumSize;
+//       public double DollarNotional;
+//       public bool HasData;
+//       public DateTime SessionDay;
+//       public bool InitialSnapshotEmitted;
+//    }
+
+//    readonly ViewStream _priceStream;
+//    readonly BarType _barType;
+//    readonly int _barSize;
+//    readonly PriceSource _priceSource;
+//    readonly VolumeSource _volumeSource;
+//    readonly string _timestampCol;
+//    readonly bool _emitForming;
+//    readonly bool _emitInitialSnapshot;
+//    readonly SessionReset _sessionReset;
+
+//    readonly Dictionary<string, BarState> _stateByContract = new(StringComparer.Ordinal);
+//    readonly Dictionary<string, bool> _validCols = new(StringComparer.OrdinalIgnoreCase);
+//    bool _validated;
+//    int _nextId;
+//    readonly BlueTable _outTable;
+//    readonly IBlueTableSchema _outSchema;
+
+//    public BarBuilderStream(
+//       ViewStream price_stream,
+//       string bar_type,
+//       int bar_size,
+//       string price_source,
+//       string volume_source,
+//       string timestamp_col = "ts_event",
+//       int emit_forming_i = 0,
+//       string session_reset = "NONE",
+//       int emit_initial_snapshot_i = 0
+//    )
+//    {
+//       _priceStream = price_stream;
+//       _barType = ParseBarType(bar_type);
+//       _barSize = bar_size;
+//       _priceSource = ParsePriceSource(price_source);
+//       _volumeSource = ParseVolumeSource(volume_source);
+//       _timestampCol = timestamp_col;
+//       _emitForming = emit_forming_i == 1;
+//       _emitInitialSnapshot = emit_initial_snapshot_i == 1;
+//       _sessionReset = ParseSessionReset(session_reset);
+
+//       if (_barSize <= 0)
+//       {
+//          throw new ArgumentException("bar_size must be > 0");
+//       }
+
+//       _outSchema = BlueDataUtility.form_schema(
+//          "BarBuilder",
+//          Tuple.Create("id", TypeCode.Int32),
+//          Tuple.Create("ts_event", TypeCode.String),
+//          Tuple.Create("bar_start_ts", TypeCode.String),
+//          Tuple.Create("bar_end_ts", TypeCode.String),
+//          Tuple.Create("contract", TypeCode.String),
+//          Tuple.Create("open", TypeCode.Double),
+//          Tuple.Create("high", TypeCode.Double),
+//          Tuple.Create("low", TypeCode.Double),
+//          Tuple.Create("close", TypeCode.Double),
+//          Tuple.Create("volume", TypeCode.Double),
+//          Tuple.Create("trade_count", TypeCode.Int32),
+//          Tuple.Create("vwap", TypeCode.Double),
+//          Tuple.Create("is_forming", TypeCode.Boolean)
+//       );
+//       _outTable = new BlueTable(_outSchema);
+//    }
+
+//    protected override void myOnFirstSubscriber()
+//    {
+//       _priceStream.onChange += _on_price_change;
+//    }
+
+//    protected override void myOnLastSubscriber()
+//    {
+//       _priceStream.onChange -= _on_price_change;
+//    }
+
+//    void _on_price_change(object newValue)
+//    {
+//       var dp = newValue as IDataPacket;
+//       if (dp is null)
+//       {
+//          return;
+//       }
+//       if (!_validated)
+//       {
+//          ValidateInputSchema(dp.newRows.schema);
+//          _validated = true;
+//       }
+
+//       var emitted = new List<IBlueRow>();
+//       foreach (IBlueRow row in dp.addedRows)
+//       {
+//          ProcessEvent(row, emitted);
+//       }
+
+//       if (emitted.Count == 0)
+//       {
+//          return;
+//       }
+//       foreach (var r in emitted)
+//       {
+//          _outTable.addRow(r);
+//       }
+//       var outDp = BlueDataUtility.build_dp_inc(_outTable, myLastValue as IDataPacket);
+//       myFireEvent(outDp);
+//    }
+
+//    void ValidateInputSchema(IBlueTableSchema schema)
+//    {
+//       _validCols.Clear();
+//       foreach (var name in schema.columnNames)
+//       {
+//          _validCols[name] = true;
+//       }
+
+//       if (!_validCols.ContainsKey(_timestampCol))
+//       {
+//          throw new ArgumentException($"Missing required column: {_timestampCol}");
+//       }
+
+//       switch (_priceSource)
+//       {
+//          case PriceSource.trade_price:
+//             RequireCol("trade_price");
+//             break;
+//          case PriceSource.mid:
+//             RequireCol("bid_price");
+//             RequireCol("ask_price");
+//             break;
+//          case PriceSource.bid:
+//             RequireCol("bid_price");
+//             break;
+//          case PriceSource.ask:
+//             RequireCol("ask_price");
+//             break;
+//       }
+
+//       if (_barType is BarType.VOLUME or BarType.DOLLAR)
+//       {
+//          if (_volumeSource == VolumeSource.none || !_validCols.ContainsKey("trade_size"))
+//          {
+//             throw new ArgumentException("VOLUME/DOLLAR require executed trade volume; invalid for BBO-only streams.");
+//          }
+//       }
+//       else if (_volumeSource == VolumeSource.trade_size)
+//       {
+//          RequireCol("trade_size");
+//       }
+//    }
+
+//    void RequireCol(string name)
+//    {
+//       if (!_validCols.ContainsKey(name))
+//       {
+//          throw new ArgumentException($"Missing required column for selected source: {name}");
+//       }
+//    }
+
+//    void ProcessEvent(IBlueRow row, List<IBlueRow> emitted)
+//    {
+//       var ts = ParseTimestamp(row);
+//       var contract = ResolveContract(row);
+//       var price = ResolvePrice(row);
+//       var size = ResolveSize(row);
+//       var state = GetOrCreateState(contract);
+
+//       if (state.HasData && _sessionReset != SessionReset.NONE && ts.Date != state.SessionDay.Date)
+//       {
+//          emitted.Add(ToBarRow(state, ts_event: state.EndTs, isForming: false));
+//          state.HasData = false;
+//       }
+
+//       switch (_barType)
+//       {
+//          case BarType.TIME:
+//             ProcessTimeBar(state, ts, price, size, emitted);
+//             break;
+//          case BarType.TICK:
+//             ProcessThresholdBar(state, ts, price, size, emitted, state.TradeCount + 1 >= _barSize);
+//             break;
+//          case BarType.VOLUME:
+//             ProcessThresholdBar(state, ts, price, size, emitted, state.Volume + size >= _barSize);
+//             break;
+//          case BarType.DOLLAR:
+//             ProcessThresholdBar(state, ts, price, size, emitted, state.DollarNotional + (price * size) >= _barSize);
+//             break;
+//       }
+//    }
+
+//    void ProcessTimeBar(BarState state, DateTime ts, double price, double size, List<IBlueRow> emitted)
+//    {
+//       var bucketStart = AlignTimeBucket(ts, _barSize);
+//       var bucketEnd = bucketStart.AddSeconds(_barSize);
+
+//       if (!state.HasData)
+//       {
+//          StartBar(state, ts, bucketStart, bucketEnd, price, size);
+//          MaybeEmitInitialSnapshot(state, ts, emitted);
+//          if (_emitForming)
+//          {
+//             emitted.Add(ToBarRow(state, ts, isForming: true));
+//          }
+//          return;
+//       }
+
+//       if (bucketStart < state.StartTs)
+//       {
+//          Log.Gen.error(
+//             $"BarBuilderStream ignored out-of-order event. contract={state.Contract} ts={ToIso(ts)} " +
+//             $"current_bar_start={ToIso(state.StartTs)}");
+//          return;
+//       }
+
+//       if (bucketStart > state.StartTs)
+//       {
+//          emitted.Add(ToBarRow(state, state.EndTs, isForming: false));
+//          StartBar(state, ts, bucketStart, bucketEnd, price, size);
+//          if (_emitForming)
+//          {
+//             emitted.Add(ToBarRow(state, ts, isForming: true));
+//          }
+//          return;
+//       }
+
+//       Accumulate(state, price, size);
+//       state.EndTs = bucketEnd;
+//       if (_emitForming)
+//       {
+//          emitted.Add(ToBarRow(state, ts, isForming: true));
+//       }
+//    }
+
+//    void ProcessThresholdBar(
+//       BarState state,
+//       DateTime ts,
+//       double price,
+//       double size,
+//       List<IBlueRow> emitted,
+//       bool shouldClose
+//    )
+//    {
+//       if (!state.HasData)
+//       {
+//          StartBar(state, ts, ts, ts, price, size);
+//          MaybeEmitInitialSnapshot(state, ts, emitted);
+//       }
+//       else
+//       {
+//          Accumulate(state, price, size);
+//       }
+//       state.EndTs = ts;
+
+//       if (_emitForming)
+//       {
+//          emitted.Add(ToBarRow(state, ts, isForming: true));
+//       }
+//       if (shouldClose)
+//       {
+//          emitted.Add(ToBarRow(state, ts, isForming: false));
+//          state.HasData = false;
+//       }
+//    }
+
+//    static DateTime AlignTimeBucket(DateTime ts, int seconds)
+//    {
+//       var utc = ts.ToUniversalTime();
+//       long bucketTicks = TimeSpan.FromSeconds(seconds).Ticks;
+//       long aligned = (utc.Ticks / bucketTicks) * bucketTicks;
+//       return new DateTime(aligned, DateTimeKind.Utc);
+//    }
+
+//    void MaybeEmitInitialSnapshot(BarState state, DateTime ts, List<IBlueRow> emitted)
+//    {
+//       if (!_emitInitialSnapshot || state.InitialSnapshotEmitted)
+//       {
+//          return;
+//       }
+//       emitted.Add(ToBarRow(state, ts, isForming: false));
+//       state.InitialSnapshotEmitted = true;
+//    }
+
+//    void StartBar(
+//       BarState state,
+//       DateTime ts,
+//       DateTime barStart,
+//       DateTime barEnd,
+//       double price,
+//       double size
+//    )
+//    {
+//       state.StartTs = barStart;
+//       state.EndTs = barEnd;
+//       state.Open = price;
+//       state.High = price;
+//       state.Low = price;
+//       state.Close = price;
+//       state.Volume = 0;
+//       state.TradeCount = 0;
+//       state.SumPriceSize = 0;
+//       state.SumSize = 0;
+//       state.DollarNotional = 0;
+//       state.SessionDay = ts.Date;
+//       state.HasData = true;
+//       Accumulate(state, price, size);
+//    }
+
+//    static void Accumulate(BarState state, double price, double size)
+//    {
+//       state.High = Math.Max(state.High, price);
+//       state.Low = Math.Min(state.Low, price);
+//       state.Close = price;
+//       state.TradeCount++;
+//       if (size > 0)
+//       {
+//          state.Volume += size;
+//          state.SumSize += size;
+//          state.SumPriceSize += price * size;
+//          state.DollarNotional += price * size;
+//       }
+//    }
+
+//    IBlueRow ToBarRow(BarState state, DateTime ts_event, bool isForming)
+//    {
+//       double vwap = state.SumSize > 0 ? state.SumPriceSize / state.SumSize : state.Close;
+//       var row = new MutableBlueRow(_outSchema);
+//       row.setValue("id", _nextId++);
+//       row.setValue("ts_event", ToIso(ts_event));
+//       row.setValue("bar_start_ts", ToIso(state.StartTs));
+//       row.setValue("bar_end_ts", ToIso(state.EndTs));
+//       row.setValue("contract", state.Contract);
+//       row.setValue("open", state.Open);
+//       row.setValue("high", state.High);
+//       row.setValue("low", state.Low);
+//       row.setValue("close", state.Close);
+//       row.setValue("volume", _volumeSource == VolumeSource.none ? 0.0 : state.Volume);
+//       row.setValue("trade_count", state.TradeCount);
+//       row.setValue("vwap", vwap);
+//       row.setValue("is_forming", isForming);
+//       return row;
+//    }
+
+//    BarState GetOrCreateState(string contract)
+//    {
+//       if (_stateByContract.TryGetValue(contract, out var existing))
+//       {
+//          return existing;
+//       }
+//       var created = new BarState { Contract = contract };
+//       _stateByContract[contract] = created;
+//       return created;
+//    }
+
+//    DateTime ParseTimestamp(IBlueRow row)
+//    {
+//       object? raw = row[_timestampCol];
+//       if (raw is DateTime dt)
+//       {
+//          return dt;
+//       }
+//       if (raw is string s && DateTime.TryParse(s, null, DateTimeStyles.RoundtripKind, out dt))
+//       {
+//          return dt;
+//       }
+//       throw new ArgumentException($"Invalid timestamp value in column '{_timestampCol}'.");
+//    }
+
+//    string ResolveContract(IBlueRow row)
+//    {
+//       if (_validCols.ContainsKey("contract"))
+//       {
+//          return (row["contract"]?.ToString() ?? string.Empty).Trim();
+//       }
+//       if (_validCols.ContainsKey("symbol"))
+//       {
+//          return (row["symbol"]?.ToString() ?? string.Empty).Trim();
+//       }
+//       return string.Empty;
+//    }
+
+//    double ResolvePrice(IBlueRow row)
+//    {
+//       return _priceSource switch
+//       {
+//          PriceSource.trade_price => ToDouble(row["trade_price"], "trade_price"),
+//          PriceSource.mid => (ToDouble(row["bid_price"], "bid_price") + ToDouble(row["ask_price"], "ask_price")) / 2.0,
+//          PriceSource.bid => ToDouble(row["bid_price"], "bid_price"),
+//          PriceSource.ask => ToDouble(row["ask_price"], "ask_price"),
+//          _ => throw new ArgumentOutOfRangeException()
+//       };
+//    }
+
+//    double ResolveSize(IBlueRow row)
+//    {
+//       if (_volumeSource == VolumeSource.none)
+//       {
+//          return 0;
+//       }
+//       return ToDouble(row["trade_size"], "trade_size");
+//    }
+
+//    static double ToDouble(object? value, string colName)
+//    {
+//       var d = BlueDataUtility.GetDouble(value!);
+//       if (!d.HasValue)
+//       {
+//          throw new ArgumentException($"Column '{colName}' must be numeric.");
+//       }
+//       return d.Value;
+//    }
+
+//    static string ToIso(DateTime dt) =>
+//       dt.ToUniversalTime().ToString("O");
+
+//    static BarType ParseBarType(string value)
+//    {
+//       if (Enum.TryParse<BarType>(value, true, out var parsed))
+//       {
+//          return parsed;
+//       }
+//       throw new ArgumentException("bar_type must be one of: TIME | TICK | VOLUME | DOLLAR");
+//    }
+
+//    static PriceSource ParsePriceSource(string value)
+//    {
+//       if (Enum.TryParse<PriceSource>(value, true, out var parsed))
+//       {
+//          return parsed;
+//       }
+//       throw new ArgumentException("price_source must be one of: trade_price | mid | bid | ask");
+//    }
+
+//    static VolumeSource ParseVolumeSource(string value)
+//    {
+//       if (Enum.TryParse<VolumeSource>(value, true, out var parsed))
+//       {
+//          return parsed;
+//       }
+//       throw new ArgumentException("volume_source must be one of: trade_size | none");
+//    }
+
+//    static SessionReset ParseSessionReset(string value)
+//    {
+//       if (!Enum.TryParse<SessionReset>(value, true, out var parsed))
+//       {
+//          throw new ArgumentException("session_reset must be one of: NONE | DAY | RTH | ETH");
+//       }
+//       // TODO: differentiate RTH/ETH session boundaries; currently treated as DAY.
+//       if (parsed is SessionReset.RTH or SessionReset.ETH)
+//       {
+//          return SessionReset.DAY;
+//       }
+//       return parsed;
+//    }
+// }

src/DataStreamSuite/BlueOrbOperatorStreams/BlueOrbOperatorStreams.csproj

@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
+    <PackageReference Include="OpenAI" Version="2.0.0" />
+    <PackageReference Include="System.Linq.Async" Version="6.0.1" />
+  </ItemGroup>
+
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\..\cs\Bts.Spark.Api\Bts.Spark.Api.csproj" />
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+        <ProjectReference Include="..\DataStream\DataStream.csproj" />
+        <ProjectReference Include="..\PubSub\PubSub.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="Bts.Spark.Api" Version="$(SparkVersion)" />
+        <PackageReference Include="DataStream" Version="$(SparkVersion)" />
+        <PackageReference Include="PubSub" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
+
+  <!-- <ItemGroup>
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubHeartbeatSchema.json" TypeName="PubSubHeartbeatRow" TableName="PubSubHeartbeat" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubRouteSchema.json" TypeName="PubSubRouteRow" TableName="PubSubRoute" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubServerStateSchema.json" TypeName="PubSubServerStateRow" TableName="PubSubServerState" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubServerTagStatsSchema.json" TypeName="PubSubServerTagStatsRow" TableName="PubSubServerTagStats" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\TableMetadataSchema.json" TypeName="TableMetadataRow" TableName="TableMetadata" />
+  </ItemGroup> -->
+
+</Project>

src/DataStreamSuite/BlueOrbOperatorStreams/Data/ContractUtils.cs

@@ -0,0 +1,294 @@
+namespace BlueOrb.Internal
+{
+   using bts;
+   //using bts.api;
+   //using bts.spark;
+
+   using System;
+   using System.Diagnostics.CodeAnalysis;
+
+   public static class ContractUtils
+   {
+      public record FutureTradableContract(
+         string Symbol,
+         DateTime Expiration,
+         Exchange Exchange,
+         string? ContractText);
+
+      public class FutureSpreadContract
+      {
+         public List<Tuple<double, FutureTradableContract>> legs = new(2);
+         public string Symbol
+         {
+            get
+            {
+               return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.Symbol;
+            }
+         }
+         public string? ContractText;
+      }
+
+      public record FutureContinousContract(string Symbol, Exchange Exchange);
+
+      // this is for all future conytracts belonging to a symbol
+      public record FutureWildcardContract(string Symbol, Exchange Exchange);
+
+      public class FutureContract
+      {
+         public FutureWildcardContract? wildcard_contract;
+         public FutureTradableContract? future_tradable_contract;
+         public FutureSpreadContract? spread_tradable_contract;
+         public FutureContinousContract? continous_constract;
+
+         public FutureContract(string txt)
+         {
+            wildcard_contract = parse_wildcard_contract(txt);
+            if (wildcard_contract != null)
+               return;
+            var fc = parse_future_contract(txt);
+            if (fc != null)
+            {
+               future_tradable_contract = fc;
+               return;
+            }
+            var sc = parse_future_spread_contract(txt);
+            if (sc != null)
+            {
+               spread_tradable_contract = sc;
+               return;
+            }
+            // this is not totally true, since ParseFutureSpreadContract does not
+            // parse ICS spreads
+            continous_constract = new(txt, Exchange.CME);
+         }
+
+         [MemberNotNullWhen(true, nameof(continous_constract))]
+         public bool is_continous_contract => continous_constract != null;
+
+         [MemberNotNullWhen(true, nameof(wildcard_contract))]
+         public bool is_wildcard_contract => wildcard_contract != null;
+
+         [MemberNotNullWhen(true, nameof(future_tradable_contract))]
+         public bool is_future_tradable_contract => future_tradable_contract != null;
+
+         [MemberNotNullWhen(true, nameof(spread_tradable_contract))]
+         public bool is_spread_tradable_contract => spread_tradable_contract != null;
+
+         public string symbol()
+         {
+            if (wildcard_contract != null)
+               return wildcard_contract.Symbol;
+            if (continous_constract != null)
+            {
+               return continous_constract.Symbol;
+            }
+            else if (future_tradable_contract != null)
+            {
+               return future_tradable_contract.Symbol;
+            }
+            else if (spread_tradable_contract != null)
+            {
+               return spread_tradable_contract.Symbol;
+            }
+            Log.assert(false);
+            return string.Empty;
+         }
+         public string? contract_text
+         {
+            get
+            {
+               if (future_tradable_contract != null)
+                  return future_tradable_contract.ContractText;
+               if (spread_tradable_contract != null)
+                  return spread_tradable_contract.ContractText;
+               return null;
+            }
+         }
+      }
+      #region Parsing
+      public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME)
+      {
+         if (contract_txt.EndsWith("*"))
+         {
+            contract_txt = contract_txt.Replace("*", string.Empty);
+            return new FutureWildcardContract(contract_txt, exchange);
+         }
+         return null;
+      }
+      public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME)
+      {
+         // Validate the future string length
+         if (!contract_txt.All(c => char.IsLetterOrDigit(c)))
+            return null;
+         if (contract_txt.Length < 4)
+         {
+            return null;
+         }
+
+         // Extract the symbol and the expiration month/year codes
+         string symbol = contract_txt.Substring(0, contract_txt.Length - 2);
+         char monthCode = contract_txt[contract_txt.Length - 2];
+         char yearCode = contract_txt[contract_txt.Length - 1];
+
+         // Map month code to month, return null if the month code is invalid
+         int month = monthCode switch
+         {
+            'F' => 1,  // January
+            'G' => 2,  // February
+            'H' => 3,  // March
+            'J' => 4,  // April
+            'K' => 5,  // May
+            'M' => 6,  // June
+            'N' => 7,  // July
+            'Q' => 8,  // August
+            'U' => 9,  // September
+            'V' => 10, // October
+            'X' => 11, // November
+            'Z' => 12, // December
+            _ => 0 // Return 0 for invalid month code
+         };
+
+         if (month == 0)
+         {
+            return null; // Invalid month code
+         }
+
+         // Calculate the year based on the current decade
+         int currentYear = DateTime.Now.Year;
+         int decadeStart = currentYear / 10 * 10;
+         int year = decadeStart + (yearCode - '0');
+
+         // If the year is in the past, assume the next decade
+         if (year < currentYear)
+         {
+            year += 10;
+         }
+
+         // Create the FutureContract instance and set its properties
+         DateTime expiration = new DateTime(year, month, 1);
+         return new(symbol, expiration, exchange, contract_txt);
+      }
+      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
+      public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME)
+      {
+         // Split the string by '-' to handle each leg
+         var spreadParts = contract_txt.Split('-');
+         if (spreadParts.Length < 2)
+         {
+            return null; // Not a valid spread format
+         }
+
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };
+
+         foreach (var part in spreadParts)
+         {
+            // Split each leg by '*', assuming the format is weight*future or just future
+            string[] legParts = part.Split('*');
+            double weight = 1.0;
+            string futurePart;
+
+            if (legParts.Length == 2)
+            {
+               if (!double.TryParse(legParts[0], out weight))
+               {
+                  return null; // Invalid weight, parsing failed
+               }
+               futurePart = legParts[1];
+            }
+            else if (legParts.Length == 1)
+            {
+               futurePart = legParts[0];
+            }
+            else
+            {
+               return null; // Invalid format, too many '*' parts
+            }
+
+            // Parse the future contract
+            var futureContract = parse_future_contract(futurePart, exchange);
+            if (futureContract == null)
+            {
+               return null; // Failed to parse the future part
+            }
+
+            // Add the leg to the spread contract
+            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
+         }
+         return spreadContract;
+      }
+
+      // public static ContractType DetermineFutureOrSpread(string contract) {
+      //    if (contract.Contains("-")) {
+      //       return ContractType.FutureSpread;
+      //    } else {
+      //       return ContractType.Future;
+      //    }
+      // }
+      #endregion
+
+      public static ContractType get_type(string contract_txt)
+      {
+         var fc = parse_future_contract(contract_txt);
+         if (fc != null)
+         {
+            return ContractType.Future;
+         }
+         var sc = parse_future_spread_contract(contract_txt);
+         if (sc != null)
+         {
+            return ContractType.FutureSpread;
+         }
+         // tehre are some non calendar spreads we cant parse
+         //Log.assert(false);
+         return ContractType.FutureSpread;
+      }
+
+      public static ContractCategory get_category(string symbol)
+      {
+         var tp = get_symbol_category(symbol);
+         return tp.Item1;
+      }
+
+      public static Tuple<ContractCategory, string> get_symbol_category(string symbol)
+      {
+         var fc = parse_future_contract(symbol);
+         if (fc != null)
+         {
+            return Tuple.Create(ContractCategory.FutureString, fc.Symbol);
+         }
+         var sc = parse_future_spread_contract(symbol);
+         if (sc != null)
+         {
+            return Tuple.Create(ContractCategory.FutureString, sc.Symbol);
+         }
+         return Tuple.Create(ContractCategory.Symbol, symbol);
+      }
+
+
+
+      public enum ContractType
+      {
+         Spot,
+         Option,
+         Future,
+         FutureSpread,
+         OptionSpread
+      }
+      public enum ContractCategory
+      {
+         Symbol = 0,
+         FutureString,
+         SpreadString
+      }
+      public enum CallPut
+      {
+         Call,
+         Put
+      }
+      public enum Exchange
+      {
+         CME
+      }
+
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Data/ReplayFutureLiveBBOStream.cs

@@ -0,0 +1,99 @@
+namespace BlueOrb.Operators.Data {
+   using BCG;
+
+   using bts;
+
+   using Spark.Core.Diagnostics;
+
+   using BlueOrb.Internal;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("ReplayFutureLiveBBOStream")]
+   public class ReplayFutureLiveBBOStream : ViewStream, BCG.Utilities.General.IWorker {
+      private class LastFireTime {
+         public DateTime last_fireitme_file = DateTime.MinValue;
+         public DateTime last_firetime_clock = DateTime.MinValue;
+      }
+      private class RowsWithPointer {
+         public List<IBlueRow>? rows;
+         public int pointer;
+      }
+
+
+      readonly ViewStreamDatabase? _vsd;
+      readonly ContractUtils.FutureContinousContract? _fcc;
+      private readonly LastFireTime _last_fire_time = new LastFireTime();
+      private RowsWithPointer _rows = new RowsWithPointer();
+
+      public ReplayFutureLiveBBOStream(string continous_contract) {
+         Func<string> iso_8601_now = () => {
+            DateTime utc_now = DateTime.UtcNow;
+            long ticks = utc_now.Ticks; // 1 tick = 100 ns
+            long nanoseconds = ticks % TimeSpan.TicksPerSecond * 100;
+            string formatted = utc_now.ToString("yyyy-MM-ddTHH:mm:ss");
+            formatted += $".{nanoseconds:D9}Z";
+            return formatted;
+         };
+         ViewStream rs = new ReplayFutureLiveBBOStream(continous_contract, true);
+         IBlueTableSchema schema_out = BlueDataUtility.form_schema("BBO",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("ts_event", TypeCode.String),
+               Tuple.Create("contract", TypeCode.String),
+               Tuple.Create("bid_price", TypeCode.Double),
+               Tuple.Create("bid_size", TypeCode.Int32),
+               Tuple.Create("ask_price", TypeCode.Double),
+               Tuple.Create("ask_size", TypeCode.Int32));
+         rs = new MapStream(rs, s => schema_out,
+            r =>
+            new BlueRow(schema_out, r["id"], /*r.get<string>("ts_event")*/iso_8601_now(), r.get<string>("symbol"),
+            r.get<double>("bid_px_00"), r.get<int>("bid_sz_00"),
+            r.get<double>("ask_px_00"), r.get<int>("ask_sz_00")));
+         mySetRealSource(rs);
+      }
+
+      private ReplayFutureLiveBBOStream(string continous_contract, bool noop)
+         : this(ViewStreamDatabase.instance, new ContractUtils.FutureContract(continous_contract)) { }
+
+      public ReplayFutureLiveBBOStream(ViewStreamDatabase vsd, ContractUtils.FutureContract fc) {
+         var fcc = fc.continous_constract;
+         Log.assert(fcc != null);
+         _vsd = vsd;
+         _fcc = fcc;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         Log.notnull(_fcc);
+         var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.Symbol}_small.csv".ToLower());
+         var rows = BCG.CSVReaderStream.LoadFile(file_path);
+         _rows.rows = new(rows.Count);
+         _rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
+         _rows.pointer = 0;
+         ExceptionAssert.NotNull(_vsd).addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         ExceptionAssert.NotNull(_vsd).removeWorker(this);
+      }
+
+      public void doWork() {
+         var rows = ExceptionAssert.NotNull(_rows.rows);
+         IBlueRow cand_row = rows[_rows.pointer];
+         string cand_ts_event = cand_row.get<string>("ts_event");
+         DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
+         TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
+         TimeSpan ts_clock = DateTime.Now - _last_fire_time.last_firetime_clock;
+         if (ts_clock >= cand_ts_span_file) {
+            _rows.pointer++;
+            _last_fire_time.last_firetime_clock = DateTime.Now;
+            _last_fire_time.last_fireitme_file = cand_ts_event_dt;
+            BlueTable bt_out = new BlueTable(cand_row.schema);
+            bt_out.addRow(cand_row);
+            var dp_out = BlueDataUtility.build_dp(bt_out, myLastValue as IDataPacket);
+            myFireEvent(dp_out);
+         }
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Data/ReplayFutureLiveTradeStream.cs

@@ -0,0 +1,111 @@
+namespace Blueorb.Operators.Data
+{
+   using BCG;
+
+   using Spark.Core.Diagnostics;
+   using BlueOrb.Internal;
+   using bts;
+   using BlueOrb.Operators.Core;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("ReplayFutureLiveTradeStream")]
+   public class ReplayFutureLiveTradeStream : ViewStream, BCG.Utilities.General.IWorker
+   {
+      private class LastFireTime
+      {
+         public DateTime last_fireitme_file = DateTime.MinValue;
+         public DateTime last_firetime_clock = DateTime.MinValue;
+      }
+      private class RowsWithPointer
+      {
+         public List<IBlueRow>? rows;
+         public int pointer;
+      }
+
+
+      readonly ViewStreamDatabase? _vsd;
+      readonly ContractUtils.FutureContinousContract? _fcc;
+      private readonly LastFireTime _last_fire_time = new LastFireTime();
+      readonly RowsWithPointer _rows = new();
+
+      public ReplayFutureLiveTradeStream(string continous_contract)
+      {
+         Func<string> iso_8601_now = () =>
+         {
+            DateTime utc_now = DateTime.UtcNow;
+            long ticks = utc_now.Ticks; // 1 tick = 100 ns
+            long nanoseconds = ticks % TimeSpan.TicksPerSecond * 100;
+            string formatted = utc_now.ToString("yyyy-MM-ddTHH:mm:ss");
+            formatted += $".{nanoseconds:D9}Z";
+            return formatted;
+         };
+         ViewStream rs = new ReplayFutureLiveTradeStream(continous_contract, true);
+         IBlueTableSchema schema_out = BlueDataUtility.form_schema("BBO",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("ts_event", TypeCode.String),
+               Tuple.Create("contract", TypeCode.String),
+               Tuple.Create("trade_size", TypeCode.Double),
+               Tuple.Create("trade_price", TypeCode.Int32),
+               Tuple.Create("aggressor_side", TypeCode.String));
+         rs = new MapStream(rs, s => schema_out,
+            r =>
+            new BlueRow(schema_out, r["id"], /*r.get<string>("ts_event")*/iso_8601_now(), r.get<string>("symbol"),
+            r.get<double>("size"), r.get<double>("price"),
+            r.get<string>("side").StartsWith("B") ? "BUY" : (r.get<string>("side").StartsWith("A") ? "SELL" : "NONE")));
+         rs = new TimeSeriesStream(rs);
+         mySetRealSource(rs);
+      }
+
+      private ReplayFutureLiveTradeStream(string continous_contract, bool noop)
+         : this(ViewStreamDatabase.instance, new ContractUtils.FutureContract(continous_contract)) { }
+
+      public ReplayFutureLiveTradeStream(ViewStreamDatabase vsd, ContractUtils.FutureContract fc)
+      {
+         var fcc = fc.continous_constract;
+         Log.assert(fcc != null);
+         _vsd = vsd;
+         _fcc = fcc;
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         Log.notnull(_vsd);
+         Log.notnull(_fcc);
+         var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.Symbol}.csv");
+         var csv_rd = new BlueOrb.Operators.Core.CSVReaderStream(file_path, false);
+         var rows = BCG.CSVReaderStream.LoadFile(file_path);
+         _rows.rows = new(rows.Count);
+         _rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
+         _rows.pointer = 0;
+         ExceptionAssert.NotNull(_vsd).addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         ExceptionAssert.NotNull(_vsd).removeWorker(this);
+      }
+
+      public void doWork()
+      {
+         var rows = ExceptionAssert.NotNull(_rows.rows);
+         IBlueRow cand_row = rows[_rows.pointer];
+         string cand_ts_event = cand_row.get<string>("ts_event");
+         DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
+         TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
+         TimeSpan ts_clock = DateTime.Now - _last_fire_time.last_firetime_clock;
+         if (ts_clock >= cand_ts_span_file)
+         {
+            _rows.pointer++;
+            _last_fire_time.last_firetime_clock = DateTime.Now;
+            _last_fire_time.last_fireitme_file = cand_ts_event_dt;
+            BlueTable bt_out = new BlueTable(cand_row.schema);
+            bt_out.addRow(cand_row);
+            var dp_out = BlueDataUtility.build_dp(bt_out, myLastValue as IDataPacket);
+            myFireEvent(dp_out);
+         }
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Data/SQLiteHelper.cs

@@ -0,0 +1,144 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SQLite;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace BlueOrb.Internal
+{
+
+   public static class SQLiteHelper
+   {
+      public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+         ? "C:\\BlueOrb_Data\\"
+         : Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty, "BlueOrb_Data");
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query)
+      {
+         // Create a new DataTable to store the results
+         DataTable dataTable = new DataTable();
+
+         // Create and open a SQLite connection using the provided connection string
+         using (SQLiteConnection connection = new SQLiteConnection(connection_string))
+         {
+            try
+            {
+               connection.Open();
+
+               // Create a SQLite command with the provided query
+               using (SQLiteCommand command = new SQLiteCommand(query, connection))
+               {
+                  // Use SQLiteDataAdapter to fill the DataTable
+                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command))
+                  {
+                     adapter.Fill(dataTable);
+                  }
+               }
+            }
+            catch (Exception ex)
+            {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         // Return the populated DataTable
+         return dataTable;
+      }
+
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query)
+      {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString))
+         {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader())
+            {
+               while (reader.Read())
+               {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++)
+                  {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
+      }
+
+      public static List<Dictionary<string, object>> get_table(string connectionString, string query)
+      {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString))
+         {
+            try
+            {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader())
+               {
+                  while (reader.Read())
+                  {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++)
+                     {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            }
+            catch (Exception ex)
+            {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
+      }
+
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query)
+      {
+         using (var connection = new SQLiteConnection(connectionString))
+         {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader())
+            {
+               while (reader.Read())
+               {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++)
+                  {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
+      }
+   }
+}
+

src/DataStreamSuite/BlueOrbOperatorStreams/Internal/DSLStream.cs

@@ -0,0 +1,43 @@
+namespace BlueOrb.Internal
+{
+   using bts;
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+   using System.Xml;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream
+   {
+      private readonly string _dsl;
+      public DSLStream(string dsl)
+      {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         // Remove all colons
+         string sanitizedDsl = dsl.Replace(":", "COLON");
+         _dsl = dsl;
+         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(string stream_expression)
+      {
+         try
+         {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         }
+         catch (Exception ex)
+         {
+            //send to blueorb
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Internal/MillisecondStream.cs

@@ -0,0 +1,103 @@
+using System;
+
+using BCG.Utilities.General;
+using BCG;
+using bts;
+
+namespace BlueOrb.Internal
+{
+   [DataStreamUserRespresentation("MillisecondStream")]
+   public class MillisecondStream : ViewStream, IWorker
+   {
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private readonly TimeSpan _period;
+      private readonly int _size;
+      private readonly ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema;
+      private bts.CircularBufferTable _table;
+
+      public MillisecondStream(int interval_msecs, int size)
+      : this(ViewStreamDatabase.instance, interval_msecs, size)
+      {
+      }
+
+      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size)
+      {
+         _vsd = vsd;
+         _period = TimeSpan.FromMilliseconds(interval_msecs);
+         _size = size;
+         _schema = BlueDataUtility.form_schema("Milliseconds",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("time", TypeCode.DateTime),
+               Tuple.Create("display_time", TypeCode.String));
+         _table = new CircularBufferTable(_schema, _size);
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         _vsd.removeWorker(this);
+      }
+
+      private int _id = 0;
+      public void doWork()
+      {
+         if (DateTime.Now - _last_fire_time >= _period)
+         {
+            _last_fire_time = DateTime.Now;
+            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
+            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
+            _table.add_row(nr);
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+   }
+   internal class MSecondStream : TimeUnitStream
+   {
+      public MSecondStream(int intervalInms, int size)
+         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
+
+      protected override DateTime myRoundTime(DateTime dt) { return dt; }
+   }
+
+   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper
+   {
+      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
+                                                               new ColumnInfo("id", 0, typeof(int)),
+                                                               new ColumnInfo("time", 1, typeof(string)));
+      public DisplayableMillisecondStream(int intervalInMs, int sz)
+      {
+         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
+      }
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         DateTime time = row.get_datetime("time");
+         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+         return new BlueRow(mySchema, row["id"], time_txt);
+      }
+
+      #endregion
+
+      private string myToString(DateTime dt)
+      {
+         string sep = ":";
+         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
+      }
+   }
+}

src/DataStreamSuite/BlueOrbOperatorStreams/Signals/BarBuilderStream.cs

@@ -0,0 +1,348 @@
+namespace BlueOrb.Operators.Signal
+{
+   using System;
+   using System.Collections.Concurrent;
+   using System.IO.MemoryMappedFiles;
+   using BCG;
+   using System.Text.Json.Serialization;
+   using System.Text.Json;
+   using System.Data;
+
+
+   public enum BarType
+   {
+      TIME,
+      TICK,
+      VOLUME,
+      DOLLAR,
+   }
+
+   public class BarBuilderSchemaInfo
+   {
+      [JsonPropertyName("price_col")]
+      public string price_col { get; set; } = "trade_price";
+
+      [JsonPropertyName("volume_col")]
+      public string volume_col { get; set; } = "trade_size";
+
+      [JsonPropertyName("timestamp_col")]
+      public string timestamp_col { get; set; } = "ts_event";
+   }
+
+
+   public class SingleBar : IBlueRow
+   {
+      private BarBuilderStream _parent_str;
+      private IBlueTableSchema _schema;
+      private MutableBlueRow _row;
+      public SingleBar(BarBuilderStream parent_str)
+      {
+         _parent_str = parent_str;
+         _schema = BlueDataUtility.form_schema(
+            "bar_builder_single_bar",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("ts_event", TypeCode.DateTime),
+            Tuple.Create("bar_start_ts", TypeCode.DateTime),
+            Tuple.Create("bar_end_ts", TypeCode.DateTime),
+            Tuple.Create("open", TypeCode.Double),
+            Tuple.Create("high", TypeCode.Double),
+            Tuple.Create("low", TypeCode.Double),
+            Tuple.Create("close", TypeCode.Double),
+            Tuple.Create("volume", TypeCode.Double),
+            Tuple.Create("notional", TypeCode.Double),
+            Tuple.Create("trade_count", TypeCode.Int32),
+            Tuple.Create("n_events", TypeCode.Int32),
+            Tuple.Create("vwap", TypeCode.Double),
+            Tuple.Create("is_active", TypeCode.Boolean)
+         );
+         _row = new MutableBlueRow(_schema);
+      }
+
+      public IBlueTableSchema schema => _row.schema;
+
+      public object this[int index] => _row[index];
+
+      public object this[string col_name] => _row[col_name];
+
+
+      public IBlueRow add_event(IBlueRow r)
+      {
+         // initiaizes _row
+         void reset_new_bar()
+         {
+            _row = new MutableBlueRow(_schema);
+            _row.setValue("id", _parent_str.get_new_id());
+            _row.setValue("volume", 0.0);
+            _row.setValue("notional", 0.0);
+            _row.setValue("trade_count", 0);
+            _row.setValue("n_events", 0);
+            _row.setValue("is_active", true);
+         }
+
+         void absorb_into_row(IBlueRow input_row)
+         {
+            var ts = input_row.get_datetime(_parent_str._schema_info.timestamp_col, DateTime.UtcNow).ToUniversalTime();
+            var price = input_row.get<double>(_parent_str._schema_info.price_col, 0.0);
+            var volume = input_row.get<double>(_parent_str._schema_info.volume_col, 0.0);
+            var notional = price * volume;
+
+            if (_row.is_null("bar_start_ts"))
+            {
+               DateTime bar_start_ts;
+               DateTime bar_end_ts;
+               if (_parent_str._bar_type == BarType.TIME)
+               {
+                  var bucket_ticks = (long)(Math.Max(1.0, _parent_str._bar_size) * TimeSpan.TicksPerSecond);
+                  var start_ticks = (ts.Ticks / bucket_ticks) * bucket_ticks;
+                  bar_start_ts = new DateTime(start_ticks, DateTimeKind.Utc);
+                  bar_end_ts = bar_start_ts.AddTicks(bucket_ticks);
+               }
+               else
+               {
+                  bar_start_ts = ts;
+                  bar_end_ts = ts;
+               }
+               _row.setValue("bar_start_ts", bar_start_ts);
+               _row.setValue("bar_end_ts", bar_end_ts);
+               _row.setValue("open", price);
+               _row.setValue("high", price);
+               _row.setValue("low", price);
+            }
+            else
+            {
+               var high = _row.get<double>("high", price);
+               var low = _row.get<double>("low", price);
+               _row.setValue("high", Math.Max(high, price));
+               _row.setValue("low", Math.Min(low, price));
+            }
+
+            _row.setValue("ts_event", ts);
+            _row.setValue("close", price);
+            _row.setValue("volume", _row.get<double>("volume", 0.0) + volume);
+            _row.setValue("notional", _row.get<double>("notional", 0.0) + notional);
+            _row.setValue("trade_count", _row.get<int>("trade_count", 0) + 1);
+            _row.setValue("n_events", _row.get<int>("n_events", 0) + 1);
+            var running_volume = _row.get<double>("volume", 0.0);
+            var running_notional = _row.get<double>("notional", 0.0);
+            _row.setValue("vwap", running_volume > 0.0 ? running_notional / running_volume : price);
+         }
+
+         bool is_closed()
+         {
+            bool closed;
+            if (_parent_str._bar_type == BarType.TIME)
+            {
+               var ts = _row.get_datetime("ts_event", DateTime.MinValue);
+               var bar_end_ts = _row.get_datetime("bar_end_ts", DateTime.MaxValue);
+               closed = ts >= bar_end_ts;
+            }
+            else if (_parent_str._bar_type == BarType.TICK)
+            {
+               closed = _row.get<int>("n_events", 0) >= _parent_str._bar_size;
+            }
+            else if (_parent_str._bar_type == BarType.VOLUME)
+            {
+               closed = _row.get<double>("volume", 0.0) >= _parent_str._bar_size;
+            }
+            else
+            {
+               closed = _row.get<double>("notional", 0.0) >= _parent_str._bar_size;
+            }
+            _row.setValue("is_active", !closed);
+            return closed;
+         }
+
+         if (_row == null)
+         {
+            reset_new_bar();
+         }
+         absorb_into_row(r);
+         if (is_closed())
+         {
+            IBlueRow closed_row = new BlueRow(_row);
+            reset_new_bar();
+            return closed_row;
+         }
+         return null;
+      }
+
+   }
+
+
+   public class BarState : IBlueRowCollection
+   {
+      private SingleBar _curr_row;
+      private BlueTable _state;
+
+      public BarState(BarBuilderStream parent_str)
+      {
+         _curr_row = new SingleBar(parent_str);
+         _state = new BlueTable(_curr_row.schema);
+      }
+
+      public bool add_event(IBlueRow r)
+      {
+         var closed_bar = _curr_row.add_event(r);
+         if (closed_bar is not null)
+         {
+            // curr bar is closed, only then its added
+            // when bars are evolving we dont add it
+            _state.addRow(closed_bar);
+            return true;
+         }
+         return false;
+      }
+
+      public IBlueRow this[int id] => _state[id];
+
+      public IBlueTableSchema schema => _state.schema;
+
+      public bool containsID(int id) => _state.containsID(id);
+
+      public int Count => _state.Count;
+
+      public System.Collections.IEnumerator GetEnumerator() => _state.GetEnumerator();
+   }
+
+   [DataStreamUserRespresentation("BarBuilderStream")]
+   public class BarBuilderStream : ViewStream
+   {
+      readonly ViewStream _price_stream;
+      internal readonly BarType _bar_type;
+      internal readonly double _bar_size;
+      internal readonly bool _show_open_bars;
+      internal readonly BarBuilderSchemaInfo _schema_info;
+      private BarState _state;
+
+      public BarBuilderStream(ViewStream price_stream,
+                               BarType bar_type,
+                               double bar_size,
+                               bool show_open_bars,
+                               BarBuilderSchemaInfo schema_info)
+      {
+         _price_stream = price_stream;
+         _bar_type = bar_type;
+         _bar_size = bar_size;
+         _show_open_bars = show_open_bars;
+         _schema_info = schema_info;
+         _state = new BarState(this);
+      }
+
+      int _id = 0;
+      internal int get_new_id()
+      {
+         return _id++;
+      }
+
+      public BarBuilderStream(ViewStream price_stream,
+                               string bar_type,
+                               double bar_size,
+                               string schema_info_json)
+                               :
+                               this(price_stream,
+                                     Enum.Parse<BarType>(bar_type),
+                                     bar_size,
+                                     true,
+                                    _parse_and_validate_schema_info_json(schema_info_json))
+      {
+      }
+
+      public BarBuilderStream(ViewStream price_stream,
+                               string bar_type,
+                               double bar_size,
+                               string price_col,
+                               string volume_col,
+                               string timestamp_col)
+                               :
+                               this(price_stream,
+                                     Enum.Parse<BarType>(bar_type),
+                                     bar_size,
+                                     true,
+                                     _create_and_validate_schema_info(price_col, volume_col, timestamp_col))
+      {
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _price_stream.onChange += _on_price_stream_change;
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         _price_stream.onChange -= _on_price_stream_change;
+      }
+
+      void _on_price_stream_change(object dp_obj)
+      {
+         DataPacket dp = dp_obj as DataPacket;
+         var sz_0 = _state.Count;
+         foreach (IBlueRow r in dp.addedRows)
+         {
+            _state.add_event(r);
+         }
+         if (_state.Count != sz_0)
+         // we will check for _show_open_bars here 
+         {
+            var dp_out = BlueDataUtility.build_dp_inc(_state, myLastValue as DataPacket, false);
+            myFireEvent(dp_out);
+         }
+      }
+
+
+      static BarBuilderSchemaInfo _parse_and_validate_schema_info_json(string schema_info_json)
+      {
+         if (string.IsNullOrWhiteSpace(schema_info_json))
+         {
+            throw new ArgumentException("schema_info_json cannot be null or empty.", nameof(schema_info_json));
+         }
+
+         var schema_info = JsonSerializer.Deserialize<BarBuilderSchemaInfo>(schema_info_json);
+         _validate_schema_info(schema_info);
+         return schema_info!;
+      }
+
+      static BarBuilderSchemaInfo _create_and_validate_schema_info(
+         string price_col,
+         string volume_col,
+         string timestamp_col)
+      {
+         var schema_info = new BarBuilderSchemaInfo
+         {
+            price_col = price_col,
+            volume_col = volume_col,
+            timestamp_col = timestamp_col,
+         };
+         _validate_schema_info(schema_info);
+         return schema_info;
+      }
+
+      static void _validate_schema_info(BarBuilderSchemaInfo? schema_info)
+      {
+         if (schema_info is null)
+         {
+            throw new ArgumentException("schema_info_json did not deserialize into BarBuilderSchemaInfo.");
+         }
+
+         if (string.IsNullOrWhiteSpace(schema_info.price_col))
+         {
+            throw new ArgumentException("BarBuilderSchemaInfo.PriceCol cannot be null or empty.");
+         }
+         if (string.IsNullOrWhiteSpace(schema_info.volume_col))
+         {
+            throw new ArgumentException("BarBuilderSchemaInfo.VolumeCol cannot be null or empty.");
+         }
+         if (string.IsNullOrWhiteSpace(schema_info.timestamp_col))
+         {
+            throw new ArgumentException("BarBuilderSchemaInfo.TimestampCol cannot be null or empty.");
+         }
+
+         if (schema_info.price_col.Equals(schema_info.volume_col, StringComparison.OrdinalIgnoreCase) ||
+             schema_info.price_col.Equals(schema_info.timestamp_col, StringComparison.OrdinalIgnoreCase) ||
+             schema_info.volume_col.Equals(schema_info.timestamp_col, StringComparison.OrdinalIgnoreCase))
+         {
+            throw new ArgumentException("BarBuilderSchemaInfo columns must be distinct.");
+         }
+      }
+   }
+
+}

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,23 +6,80 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var cmp1 = value1.Cast<IComparable>();
-      var cmp2 = value2.Cast<IComparable>();
+      var obj1 = value1.GetValue(env);
+      var obj2 = value2.GetValue(env);
+      if (obj1 is null) {
+         if (obj2 is null) {
+            return 0;
+         }
+         return -1;
+      }
+      if (obj2 is null) {
+         return 1;
+      }
+      return compareObjs(obj1, obj2);
+   }
+
+   static int compareObjs(object obj1, object obj2) {
       try {
-         var v1 = cmp1.GetValue(env);
-         var v2 = cmp2.GetValue(env);
-         return v1.GetType().IsNumeric() ?
-            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
+         var typ1 = obj1.GetType();
+         var typ2 = obj2.GetType();
+         bool areSameTypes = typ1 == typ2;
+         if (areSameTypes) {
+            return ((IComparable)obj1).CompareTo(obj2);
+         }
+         if (typ1.IsNumeric()) {
+            return tryCompareNumeric(obj1, obj2);
+         } else if (typ2.IsNumeric()) {
+            return -tryCompareNumeric(obj2, obj1);
+         } else if (typ1.IsEnum) {
+            if (typ2 == typeof(string)) {
+               return -((string)obj2).CompareTo(obj1.ToString());
+            }
+         } else if (typ1 == typeof(string)) {
+            return ((string)obj1).CompareTo(obj2.ToString());
+         } else if (typ1 == typeof(DateTime)) {
+            if (typ2 == typeof(TimeSpan)) {
+               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
+            }
+         } else if (typ1 == typeof(TimeSpan)) {
+            if (typ2 == typeof(DateTime)) {
+               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
+            }
+         }
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
+      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
+   }
+
+   static int tryCompareNumeric(object numericObj1, object obj2) {
+      var typ2 = obj2.GetType();
+      if (typ2.IsNumeric()) {
+         return new NumericComparer().Compare(numericObj1, obj2);
+      } else if (typ2.IsEnum) {
+         var undTyp2 = typ2.GetEnumUnderlyingType();
+         if (undTyp2 == typeof(byte)) {
+            return new NumericComparer().Compare(numericObj1, (byte)obj2);
+         }
+         if (undTyp2 == typeof(short)) {
+            return new NumericComparer().Compare(numericObj1, (short)obj2);
+         }
+         if (undTyp2 == typeof(int)) {
+            return new NumericComparer().Compare(numericObj1, (int)obj2);
+         }
+         if (undTyp2 == typeof(long)) {
+            return new NumericComparer().Compare(numericObj1, (long)obj2);
+         }
+      }
+      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -11,10 +11,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
   
   <ItemGroup>
     <Scanner Include="BooleanExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -10,11 +10,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnExpression">
     <Scanner Include="ColumnExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -10,11 +10,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnSortExpression">
     <Scanner Include="ColumnSortExpressionTokens.lex" />

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -10,9 +10,18 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnMapExpression">
     <Scanner Include="ColumnMapExpressionTokens.lex" />

src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs

@@ -1,118 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts.collections;
-using bts.utils;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-public class BlueAddIndexColumnTable : IBlueRowCollection {
-   readonly IBlueTableSchema schema;
-   readonly IList<object?>[] columnValues;
-   readonly int count;
-
-   BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
-      this.schema = schema;
-      this.columnValues = columnValues;
-      this.count = count ?? columnValues[0].Count;
-   }
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public int Count => count;
-
-   public IBlueRow this[int id] =>
-      id > 0 && id <= count
-         ? GetRow(id - 1)
-         : throw new KeyNotFoundException();
-
-   public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
-      if (columnInfos.Any(c => c.colName == "id")) {
-         throw new ArgumentException("'id' column already exists", nameof(columnInfos));
-      }
-      if (columnInfos.Length != columnValues.Length) {
-         throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
-      }
-      int count = columnValues.FirstOrDefault()?.Count ?? 0;
-      var origCols = columnInfos;
-      var origColValues = columnValues;
-      columnInfos = new ColumnInfo[origCols.Length + 1];
-      columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
-      columnValues = new IList<object?>[origCols.Length + 1];
-      columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
-      for (int i = 1; i < columnInfos.Length; i++) {
-         columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
-         columnValues[i] = origColValues[i - 1];
-      }
-      var schema = new BlueTableSchema(columnInfos, tableName);
-      return new(schema, columnValues, count);
-   }
-
-   public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
-      int count = columnValues.FirstOrDefault()?.Count ?? 0;
-      int? idColIdx = schema.TryResolveIndices("id");
-      var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
-      var columnInfos = schema.columnInfo;
-      if (idColIdx.HasValue) {
-         // Replace existing id column. columnValues may or may not contain the old ids (if its
-         // length is one less column than that of columnInfos, it doesn't contain them).
-         switch (columnValues.Length - schema.columnInfo.Length) {
-            case 0:
-               columnValues[idColIdx.Value] = idColValues;
-               break;
-            case -1:
-               columnValues = ArrayHelper.InsertExact(columnValues, idColIdx.Value, true, idColValues);
-               break;
-            default:
-               throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
-         }
-      } else {
-         if (columnValues.Length != columnInfos.Length) {
-            throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
-         }
-         var origColInfos = columnInfos;
-         columnInfos = new ColumnInfo[columnInfos.Length + 1];
-         columnInfos[0] = new("id", 0, typeof(int));
-         for (int i = 1; i < columnInfos.Length; i++) {
-            var c = origColInfos[i - 1];
-            columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
-         }
-         columnValues = ArrayHelper.InsertExact(columnValues, 0, true, idColValues);
-      }
-      return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
-   }
-
-   public bool containsID(int id) =>
-      id > 0 && id <= count;
-
-   public IList<object?> GetColumn(int index) =>
-      columnValues[index];
-
-   public IBlueRow GetRow(int index) =>
-      new BlueRowColumnWrapper(schema, index, columnValues);
-
-   public int IndexOf(IBlueRow item) {
-      int id = item.Id();
-      return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
-         ? id - 1
-         : -1;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      int count = Count;
-      for (int i = 0; i < count; i++) {
-         yield return GetRow(i);
-      }
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}

src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs

@@ -1,56 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using mbts.utils;
-
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-public class BlueIndexedColumnTable : IBlueRowCollection {
-   readonly IBlueTableSchema schema;
-   readonly IList<object?>[] columnValues;
-   readonly Dictionary<int, int> rowIdToIndex;
-
-   public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
-      this.schema = schema;
-      this.columnValues = columnValues;
-      int idColumnIndex = schema.ResolveIndices("id");
-      rowIdToIndex = columnValues[idColumnIndex]
-         .Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
-         .ToDictionary();
-   }
-
-   public IBlueRow this[int id] =>
-      rowIdToIndex.TryGetValue(id, out var idx)
-         ? GetRow(idx)
-         : throw new KeyNotFoundException();
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public int Count => rowIdToIndex.Count;
-
-   public bool containsID(int id) =>
-      rowIdToIndex.ContainsKey(id);
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      int count = Count;
-      for (int i = 0; i < count; i++) {
-         yield return GetRow(i);
-      }
-   }
-
-   public IBlueRow GetRow(int index) =>
-      new BlueRowColumnWrapper(schema, index, columnValues);
-
-   public int IndexOf(IBlueRow item) =>
-      rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
-         ? idx
-         : -1;
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}

src/DataStreamSuite/DataStream/BlueOrb/BlueRowColumnWrapper.cs

@@ -1,21 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using System.Collections.Generic;
-
-public class BlueRowColumnWrapper : IBlueRow {
-   readonly int index;
-   readonly IList<object?>[] columns;
-
-   public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
-      this.schema = schema;
-      this.index = index;
-      this.columns = columns;
-   }
-
-   public object? this[int index] =>
-      columns[index][this.index];
-
-   public IBlueTableSchema schema { get; }
-}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,9 +1,7 @@
-using System;
-
 namespace BCG {
    [DataStreamUserRespresentation("CountStream")]
    public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
+      IDataPacket? myLastFiredDataPacket;
       private IBlueTableSchema mySchema = new BlueTableSchema("count",
          new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("count", 1, typeof(int)));
@@ -14,6 +12,9 @@ namespace BCG {
          myVS = vs;
       }
 
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
+
       protected override void myOnFirstSubscriber() {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
@@ -21,7 +22,6 @@ namespace BCG {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
-
       private void myHandleChanges(object newValue) {
          myFireEvent(
             myLastFiredDataPacket =
@@ -36,11 +36,8 @@ namespace BCG {
 
       private IBlueRowCollection myGetRows(object dp, int id) {
          BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+         retVal.addRow(new BlueRow(mySchema, id, ((IDataPacket)dp).newRows.Count));
          return retVal;
       }
-
-
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -15,6 +15,7 @@ namespace bts.BlueOrb {
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_year");
             return rv;
          };
          ViewStream rs = new MapStream(parent_stream,
@@ -30,6 +31,7 @@ namespace bts.BlueOrb {
             rv.setValue($"{inp_col}_hour", dt.Hour);
             rv.setValue($"{inp_col}_minute", dt.Minute);
             rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            rv.setValue($"{inp_col}_year", dt.Date.Year);
             return rv;
          });
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,5 +1,8 @@
 namespace bts.BlueOrb {
    using BCG;
+
+   using Bts.DataStreamCore;
+
    using System;
    using System.Text.RegularExpressions;
 
@@ -21,22 +24,24 @@ namespace bts.BlueOrb {
 
          #region IStreamFilter Members
          public bool filter(IBlueRow dr) {
-            bool rv = true;
             for (int i = 0; i < myCols.Length; i++) {
-               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-               if (!rv)
-                  return rv;
+               var val = dr[myCols[i]];
+               var str = val is DBNull
+                  ? string.Empty
+                  : (val?.ToString() ?? string.Empty);
+               if (!myRegExps[i].IsMatch(str)) {
+                  return false;
+               }
             }
-            return rv;
+            return true;
          }
          #endregion
       }
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
+      BlueTable? myLastFiredRows;
+      IBlueRowCollection? myLastReceivedRows;
       private IStreamFilter myFilter;
       private ViewStream myParentStream;
       private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -52,12 +57,11 @@ namespace bts.BlueOrb {
 
 
       public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         myParentStream = vs;
+         myFilter = filter;
+         myFilterString = filter.ToString() ?? string.Empty;
+         if (filter is IDynamicStreamFilter dynFilter) {
+            dynFilter.onFilterChange += myHandleFilterChange;
          }
       }
 
@@ -86,7 +90,7 @@ namespace bts.BlueOrb {
       }
 
       protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
+         var dp = (IDataPacket)o;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -94,23 +98,27 @@ namespace bts.BlueOrb {
 
       private void myHandleCallbacks(object newValue) {
          DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
+         IDataPacket arrivedPacket = (IDataPacket)newValue;
          myLastReceivedRows = arrivedPacket.newRows;
          if (myLastFiredRows != null) {
             retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
                new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
+            var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
+            foreach (IBlueRow r in arrivedPacket.deletedRows) {
+               if (myLastFiredRows.containsID(r.Id())) {
                   myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
+                  deletedRows.Value.addRow(r);
                }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
+            }
+            var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
+            foreach (IBlueRow r in arrivedPacket.addedRows) {
                if (myFilter.filter(r)) {
                   myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
+                  addedRows.Value.addRow(r);
                }
+            }
          } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            myLastFiredRows = new BlueTable(arrivedPacket.Schema);
             retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
             foreach (IBlueRow r in arrivedPacket.newRows)
                if (myFilter.filter(r))

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -1,12 +1,13 @@
-using System;
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
 
 namespace bts.BlueOrb {
    [DataStreamUserRespresentation("GroupByStream")]
@@ -14,20 +15,21 @@ namespace bts.BlueOrb {
 
       public interface INonIncGroupByEvaluator {
          // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
+         IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[]? get_group_by_cols();
       }
 
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
+         Func<string[]?> _groupby_cols_func;
          Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
                                           Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
+
+         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable? rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
                List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
@@ -39,17 +41,19 @@ namespace bts.BlueOrb {
             }
             return rv;
          }
-         public string[] get_group_by_cols() {
+
+         public string[]? get_group_by_cols() {
             return _groupby_cols_func();
          }
 
       }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
+      ViewStream? _main_stream;
+      INonIncGroupByEvaluator? _groupby_func;
+      BlueRowHasher? _row_hasher;
+      string[]? _groupby_cols;
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
+      IDisposable? subscription;
 
       public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -58,7 +62,7 @@ namespace bts.BlueOrb {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+      public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
@@ -68,34 +72,42 @@ namespace bts.BlueOrb {
       // this is the main constructor that does everything
       public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            if (String.IsNullOrEmpty(groupby_cols))
-               return null;
-            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
-         };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
             // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
             if (rv == null)
                return new List<IBlueRow>();
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
          mySetRealSource(rs);
+
+         string[]? groupby_cols_func() {
+            if (string.IsNullOrEmpty(groupby_cols)) {
+               return null;
+            }
+            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
+         }
       }
       public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
       this(parent_str, String.Empty, standard_group_by_text) {
       }
 
+      protected override void myOnFirstSubscriber() {
+         Log.isnull(subscription);
+         subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
+      }
 
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+      protected override void myOnLastSubscriber() {
+         subscription?.Dispose();
+         subscription = null;
+      }
 
       private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
+         Log.notnull(_groupby_func);
+         var dp = (IDataPacket)nv;
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -109,7 +121,7 @@ namespace bts.BlueOrb {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
          if (total_state == null)
             // mayeb returna n empty table ?
             return;
@@ -121,7 +133,10 @@ namespace bts.BlueOrb {
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         var deletedRows = myLastValue is null
+            ? new BlueTable(total_state.schema)
+            : ((IDataPacket)myLastValue).newRows;
+         myFireEvent(new DataPacket(total_state, total_state, deletedRows));
       }
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,41 +1,44 @@
 namespace bts.BlueOrb {
    using BCG;
-   using RowHash = System.Int32;
+
+   using Bts.DataStream;
+
+   using Spark.Core.Diagnostics;
+
    using System;
    using System.Collections.Generic;
+   using System.Diagnostics.CodeAnalysis;
    using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   using System.Collections;
-   using System.Data.Entity.Core.Mapping;
+
+   using RowHash = System.Int32;
 
    [DataStreamUserRespresentation("JoinStream")]
    public class SimpleJoinStream : ViewStream {
       public class JoinBookkeepingData {
          // initialized in cons
-         private readonly List<string> _join_col_names;
-         private readonly bool _is_left;
+         readonly string[] _join_col_names;
          private readonly BlueRowHasher _hasher;
          // initialized on first udpate         
-         private IBlueTableSchema _schema;
-         private int[] _join_cols;
+         IBlueTableSchema? _schema;
+         int[]? _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
 
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
-            _is_left = is_left;
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
+            _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
             _hasher = row_hasher;
          }
 
-         public IBlueTableSchema schema => _schema;
+         [NotNullIfNotNull(nameof(_schema))]
+         public IBlueTableSchema? schema => _schema;
 
          public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
 
-         public List<IBlueRow> get_matching_rows(int hash) {
-            List<IBlueRow> rv = null;
-            _hash_row_map.TryGetValue(hash, out rv);
-            return rv;
-         }
+         [MemberNotNull(nameof(_schema))]
+         public IBlueTableSchema RequireSchema() =>
+            ExceptionAssert.MemberNotNull(_schema);
+
+         public List<IBlueRow>? get_matching_rows(int hash) =>
+            _hash_row_map.GetValueOrDefault(hash);
 
          private RowHash _get_hash(IBlueRow r) {
             if (_join_cols == null || _join_cols.Length == 0) {
@@ -44,16 +47,15 @@
             return _hasher.get_hash(r, _join_cols);
          }
 
-         public void update_data(DataPacket dp) {
+         public void update_data(IDataPacket dp) {
             if (!ready) {
                _schema = dp.newRows.schema;
-               _join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+               _join_cols = _schema.ResolveIndices(_join_col_names);
             }
             _hash_row_map.Clear();
             foreach (IBlueRow r in dp.newRows) {
                int hash = _get_hash(r);
-               List<IBlueRow> hash_rows = null;
-               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
+               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
                   hash_rows = new List<IBlueRow>();
                   _hash_row_map.Add(hash, hash_rows);
                }
@@ -61,6 +63,7 @@
             }
          }
 
+         [MemberNotNullWhen(true, nameof(_schema))]
          public bool ready {
             get {
                return _schema != null;
@@ -71,8 +74,7 @@
 
       private ViewStream _ls, _rs;
       private JoinBookkeepingData _ldata, _rdata;
-      private ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema = null;
+      IBlueTableSchema? _schema;
 
       public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
          this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
@@ -85,8 +87,8 @@
          Log.assert(lj_col_list.Count == rj_col_list.Count);
          int n_cols = lj_col_list.Count;
          BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
+         _ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
+         _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
       }
 
       protected override void myOnFirstSubscriber() {
@@ -95,13 +97,13 @@
       }
 
       private void _ls_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = (IDataPacket)dp_obj;
          _ldata.update_data(dp);
          _form_and_fire();
       }
 
       private void _rs_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = (IDataPacket)dp_obj;
          _rdata.update_data(dp);
          _form_and_fire();
       }
@@ -111,20 +113,24 @@
          _rs.onChange -= _rs_onchange;
       }
 
+      [MemberNotNull(nameof(_schema))]
       private void _build_schema() {
          if (_schema != null)
             return;
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
-         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
-            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
+         var lschema = Log.notnull(_ldata.schema);
+         var rschema = Log.notnull(_rdata.schema);
+         var cis = new List<ColumnInfo> {
+            new("id", 0, TypeCode.Int32)
+         };
+         foreach (ColumnInfo ci in lschema.columnInfo) {
+            string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
-            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
+         foreach (ColumnInfo ci in rschema.columnInfo) {
+            string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
+         _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
       }
 
       private void _form_and_fire() {
@@ -133,34 +139,11 @@
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
 
-         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.schema.columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         };
-         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
-            object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         };
          // this is where the join happens
          foreach (var kv in _ldata.rows) {
             RowHash hash = kv.Key;
             List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
+            List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
 
             foreach (IBlueRow lr in l_rows) {
                if (r_rows != null) {
@@ -179,8 +162,33 @@
          foreach (object[] rval in res_rows) {
             rv.addRow(new BlueRow(_schema, rval));
          }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
          myFireEvent(dp);
+
+         object[] build_row_array(int id, IBlueRow l_row) {
+            object[] rv = new object[_schema.columnNames.Length];
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
+               rv[i] = DBNull.Value;
+            }
+            rv[0] = id;
+            copy_row(rv, l_row, true);
+            return rv;
+         }
+
+         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
+            int offset = 1;
+            JoinBookkeepingData sdata = _ldata;
+            if (is_left) {
+               offset = 1;
+               sdata = _ldata;
+            } else {
+               sdata = _rdata;
+               offset = 1 + _ldata.RequireSchema().columnInfo.Length;
+            }
+            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
+               rval[offset + i] = r[i];
+            }
+         }
       }
    }
 

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -43,7 +43,7 @@ namespace bts.BlueOrb {
             _last_fire_time = DateTime.Now;
             IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
             _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table = _table.Add(nr);
+            _table.add_row(nr);
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
             myFireEvent(dp);
          }

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -45,8 +45,8 @@ namespace bts.BlueOrb {
       }
 
       // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
+      public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count == 0) {
             return null;
          }
 
@@ -73,8 +73,8 @@ namespace bts.BlueOrb {
       }
 
       // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
+      public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
+         if (rows.Count == 0) {
             return null;
          }
          IBlueRow ir = rows.ElementAt(start_idx);
@@ -104,8 +104,8 @@ namespace bts.BlueOrb {
       // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
       // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
 
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
+      public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count == 0) {
             return null;
          }
          IBlueRow ir = rolling_rows.ElementAt(0);
@@ -156,7 +156,6 @@ namespace bts.BlueOrb {
                   double first = col_vals.First();
                   double last = col_vals.Last();
                   return ((first - last) / last) * 100.0;
-                  return col_vals.First() - col_vals.Last();
                }
             default:
                // TODO

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -1,8 +1,7 @@
 namespace bts.BlueOrb {
    using BCG;
 
-   using System.Collections.Generic;
-   using System.Net;
+   using System;
 
    [DataStreamUserRespresentation("MultiRowRollingMathStream")]
    public class MultiRowRollingMathStream : ViewStream {
@@ -24,11 +23,12 @@
       }
 
       private void _parent_str_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = (IDataPacket)dp_obj;
          var rows = dp.newRows.to_list();
-         BlueTable out_table = null;
+         BlueTable? out_table = null;
          for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
+            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
+               ?? throw new InvalidOperationException("group_by_compute returned null row");
             if (out_table == null)
                out_table = new BlueTable(out_r.schema);
             out_table.addRow(out_r);

src/DataStreamSuite/DataStream/BlueOrb/Parquet/IParquetColumnReader.cs

@@ -1,29 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts.collections;
-
-using System;
-using System.Collections.Generic;
-
-public interface IParquetColumnReader : IDisposable {
-   IList<object?> Read();
-   IList<object?> Read(int? start, int? stop);
-}
-
-public interface IParquetColumnReader<TLogical, TPhysical> : IParquetColumnReader {
-   new IList<TLogical?> Read();
-   new IList<TLogical?> Read(int? start, int? stop);
-
-   IList<object?> IParquetColumnReader.Read() =>
-      Read().WrapSelector(v => (object?)v, null);
-
-   IList<object?> IParquetColumnReader.Read(int? start, int? stop) =>
-      Read(start, stop).WrapSelector(v => (object?)v, null);
-}
-
-public interface IParquetColumnReader<T> : IParquetColumnReader<T, T> {
-   new IList<T?> Read();
-
-   IList<object?> IParquetColumnReader.Read() =>
-      Read().WrapSelector(v => (object?)v, null);
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetAdapter.cs

@@ -1,507 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts;
-using bts.collections;
-
-using mbts.utils;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-
-public static class ParquetAdapter {
-   public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) {
-      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
-      var colReader = rowGroupReader.Column(index);
-      return getColumnReader(colReader, rowCount, colType: colType);
-   }
-
-   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader) {
-      var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
-      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
-      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
-      for (int i = 0; i < colValues.Length; i++) {
-         colValues[i] = new List<object?>(rowCount);
-      }
-      int rowGroupCount = fileReader.FileMetaData.NumRowGroups;
-      for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) {
-         using var rowGroupReader = fileReader.RowGroup(rowGroup);
-         rowGroupReader.ReadColumnsTo(ref colValues, colTypes);
-      }
-      return colValues;
-   }
-
-   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) {
-      if (!start.HasValue && !stop.HasValue) {
-         return ReadColumns(fileReader);
-      }
-      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
-      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
-      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
-      start ??= 0;
-      stop ??= rowCount;
-      var sliceLength = stop.Value - start.Value;
-      for (int i = 0; i < colValues.Length; i++) {
-         colValues[i] = new List<object?>(sliceLength);
-      }
-      int numRowGroups = fileReader.FileMetaData.NumRowGroups;
-      int? numRowsPerGroup = null;
-      int rowIdx = 0;
-      for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) {
-         if (stop.Value <= rowIdx) {
-            break;
-         }
-         using var rowGroupReader = fileReader.RowGroup(rowGroup);
-         int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows);
-         numRowsPerGroup ??= numRowsInGroup;
-         if (rowGroup < numRowGroups - 1) {
-            Log.assert(numRowsInGroup == numRowsPerGroup.Value);
-         } else {
-            Log.assert(numRowsInGroup <= numRowsPerGroup.Value);
-         }
-         int startIdx = start.Value - rowIdx;
-         int stopIdx = stop.Value - rowIdx;
-         rowIdx += numRowsInGroup;
-         if (startIdx >= numRowsInGroup) {
-            continue;
-         }
-         startIdx = Math.Max(0, startIdx);
-         stopIdx = Math.Min(stopIdx, numRowsInGroup);
-         int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx);
-         Log.assert(n == (stopIdx - startIdx));
-      }
-      return colValues;
-   }
-
-   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes) {
-      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
-      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
-         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
-         columnValues[colIdx].AddRange(colReader.Read());
-      }
-      return rowCount;
-   }
-
-   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes, int? start, int? stop) {
-      if (!start.HasValue && !stop.HasValue) {
-         return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes);
-      }
-      int numRows = checked((int)rowGroupReader.MetaData.NumRows);
-      if (start.HasValue && start.Value >= numRows) {
-         return 0;
-      }
-      int? readRowCount = null;
-      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
-         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
-         var colVals = colReader.Read(start, stop);
-         readRowCount ??= colVals.Count;
-         Log.assert(readRowCount == colVals.Count);
-         columnValues[colIdx].AddRange(colVals);
-      }
-      return readRowCount ?? throw new ArgumentException("Row group has no columns", nameof(rowGroupReader));
-   }
-
-   static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) {
-      var desc = reader.ColumnDescriptor;
-      var logicalType = desc.LogicalType;
-      return logicalType.Type switch {
-         LogicalTypeEnum.Int => IntReader(),
-         LogicalTypeEnum.String => new ParquetColumnReader<string?>(reader, bufferSize),
-         LogicalTypeEnum.Date => new ParquetColumnReader<DateTime?>(reader, bufferSize),
-         LogicalTypeEnum.Decimal => DecimalReader(),
-         LogicalTypeEnum.Timestamp => TimestampReader(),
-         LogicalTypeEnum.None => DefaultReader(),
-         _ => throw new NotImplementedException(),
-      };
-
-      IParquetColumnReader DefaultReader() {
-         return desc.PhysicalType switch {
-            PhysicalType.Boolean => new ParquetColumnReader<bool?>(reader, bufferSize),
-            PhysicalType.Int32 => new ParquetColumnReader<int?>(reader, bufferSize),
-            PhysicalType.Int64 => new ParquetColumnReader<long?>(reader, bufferSize),
-            PhysicalType.Float => new ParquetColumnReader<float?>(reader, bufferSize),
-            PhysicalType.Double => new ParquetColumnReader<double?>(reader, bufferSize),
-            _ => throw new NotImplementedException(),
-         };
-      }
-
-      IParquetColumnReader TimestampReader() {
-         var tlt = ExceptionAssert.of_type<TimestampLogicalType>(logicalType);
-         return new ParquetColumnReader<DateTimeNanos?, DateTime?>(reader, dtn => dtn?.DateTime, bufferSize);
-      }
-
-      IParquetColumnReader IntReader() {
-         var ilt = ExceptionAssert.of_type<IntLogicalType>(logicalType);
-         if (!ilt.IsSigned) {
-            if (colType is null) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(byte)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, byte?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(ushort)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, ushort?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(uint)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, uint?>(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(ulong)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            }
-            throw new NotImplementedException($"Expected signed int column type, but got {colType}");
-         } else {
-            if (colType is null) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(sbyte)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<short?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  32 => new ParquetColumnReader<int?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<long?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(short)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<int?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<long?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(int)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<short?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<long?, int?>(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(long)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<short?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<int?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            }
-            throw new NotImplementedException($"Expected unsigned int column type, but got {colType}");
-         }
-      }
-
-      IParquetColumnReader DecimalReader() {
-         var dlt = ExceptionAssert.of_type<DecimalLogicalType>(logicalType);
-         return new ParquetColumnReader<double?>(reader, bufferSize);
-      }
-   }
-
-   public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) {
-      return typeCode switch {
-         TypeCode.Boolean => new ColumnWriter<bool?>(writer, bufferSize),
-         TypeCode.SByte => new ColumnWriter<sbyte?>(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int16 => new ColumnWriter<short?>(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int32 => new ColumnWriter<int?>(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int64 => new ColumnWriter<long?>(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Byte => new ColumnWriter<byte?>(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt16 => new ColumnWriter<ushort?>(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt32 => new ColumnWriter<uint?>(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt64 => new ColumnWriter<ulong?>(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Single => new ColumnWriter<float?>(writer, bufferSize),
-         TypeCode.Double => new ColumnWriter<double?>(writer, bufferSize),
-         TypeCode.DateTime => new ColumnWriter<DateTimeNanos?>(writer, bufferSize, ConvertDateTimeObj),
-         TypeCode.String => new ColumnWriter<string?>(writer, bufferSize),
-         _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
-      };
-
-      static DateTimeNanos? ConvertDateTimeObj(object? obj) {
-         if (obj is DateTime dt) {
-            return new(dt);
-         }
-         if (obj is DateTimeNanos dtn) {
-            return dtn;
-         }
-         throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos");
-      }
-   }
-
-   public static Func<object?, sbyte?> GetInt8Converter(IntLogicalType logicalType) =>
-      GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, sbyte?> GetInt8Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj as sbyte?,
-            16 => obj => obj is short v ? checked((sbyte)v) : null,
-            32 => obj => obj is int v ? checked((sbyte)v) : null,
-            64 => obj => obj is long v ? checked((sbyte)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? checked((sbyte)v) : null,
-            16 => obj => obj is ushort v ? checked((sbyte)v) : null,
-            32 => obj => obj is uint v ? checked((sbyte)v) : null,
-            64 => obj => obj is ulong v ? checked((sbyte)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, short?> GetInt16Converter(IntLogicalType logicalType) =>
-      GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, short?> GetInt16Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj as short?,
-            32 => obj => obj is int v ? checked((short)v) : null,
-            64 => obj => obj is long v ? checked((short)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? checked((short)v) : null,
-            32 => obj => obj is uint v ? checked((short)v) : null,
-            64 => obj => obj is ulong v ? checked((short)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, int?> GetInt32Converter(IntLogicalType logicalType) =>
-      GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, int?> GetInt32Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj is short v ? v : null,
-            32 => obj => obj as int?,
-            64 => obj => obj is long v ? checked((int)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? checked((int)v) : null,
-            64 => obj => obj is ulong v ? checked((int)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, long?> GetInt64Converter(IntLogicalType logicalType) =>
-      GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, long?> GetInt64Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj is short v ? v : null,
-            32 => obj => obj is int v ? v : null,
-            64 => obj => obj as long?,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? checked((long)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, byte?> GetUInt8Converter(IntLogicalType logicalType) =>
-      GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, byte?> GetUInt8Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((byte)v) : null,
-            16 => obj => obj is short v ? checked((byte)v) : null,
-            32 => obj => obj is int v ? checked((byte)v) : null,
-            64 => obj => obj is long v ? checked((byte)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj as byte?,
-            16 => obj => obj is ushort v ? checked((byte)v) : null,
-            32 => obj => obj is uint v ? checked((byte)v) : null,
-            64 => obj => obj is ulong v ? checked((byte)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, ushort?> GetUInt16Converter(IntLogicalType logicalType) =>
-      GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, ushort?> GetUInt16Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((ushort)v) : null,
-            16 => obj => obj is short v ? checked((ushort)v) : null,
-            32 => obj => obj is int v ? checked((ushort)v) : null,
-            64 => obj => obj is long v ? checked((ushort)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? checked((ushort)v) : null,
-            64 => obj => obj is ulong v ? checked((ushort)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, uint?> GetUInt32Converter(IntLogicalType logicalType) =>
-      GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, uint?> GetUInt32Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((uint)v) : null,
-            16 => obj => obj is short v ? checked((uint)v) : null,
-            32 => obj => obj is int v ? checked((uint)v) : null,
-            64 => obj => obj is long v ? checked((uint)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? checked((uint)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, ulong?> GetUInt64Converter(IntLogicalType logicalType) =>
-      GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, ulong?> GetUInt64Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((ulong)v) : null,
-            16 => obj => obj is short v ? checked((ulong)v) : null,
-            32 => obj => obj is int v ? checked((ulong)v) : null,
-            64 => obj => obj is long v ? checked((ulong)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? v : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static string ColumnEnumTypeKey(string colName) =>
-      $"{colName}.EnumType";
-
-   public static KeyValuePair<string, string> ColumnEnumTypeEntry(string colName, Type enumType) {
-      if (enumType.IsNullable()) {
-         enumType = enumType.GetGenericArguments()[0];
-      }
-      if (!enumType.IsEnum) {
-         throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType));
-      }
-      return new(ColumnEnumTypeKey(colName), enumType.ToString());
-   }
-
-   public static Type? GetColumnEnumType(IReadOnlyDictionary<string, string>? keyValueMetadata, string columnName) {
-      if (keyValueMetadata is not null) {
-         var key = ColumnEnumTypeKey(columnName);
-         if (keyValueMetadata.TryGetValue(key, out var typeName)) {
-            return Type.GetType(typeName);
-         }
-      }
-      return null;
-   }
-
-   public interface IColumnWriter : IDisposable {
-      void Write(IReadOnlyCollection<object?> values);
-   }
-
-   public interface IColumnWriter<T> : IColumnWriter {
-      void Write(IReadOnlyCollection<T?> values);
-   }
-
-   class ColumnWriter<T> : IColumnWriter<T> {
-      readonly T?[] buffer;
-      readonly LogicalColumnWriter<T> writer;
-      readonly Func<object?, T?> convert;
-
-      public ColumnWriter(ColumnWriter writer, int bufferSize)
-         : this(writer, new T?[bufferSize]) { }
-
-      public ColumnWriter(ColumnWriter writer, int bufferSize, Func<object?, T?> convert)
-         : this(writer, new T?[bufferSize], convert) { }
-
-      public ColumnWriter(ColumnWriter writer, T?[] buffer)
-         : this(writer, buffer, obj => (T?)obj) { }
-
-      public ColumnWriter(ColumnWriter writer, T?[] buffer, Func<object?, T?> convert) {
-         this.buffer = buffer;
-         this.writer = writer.LogicalWriter<T>();
-         this.convert = convert;
-      }
-
-      public void Write(IReadOnlyCollection<T?> values) {
-         if (buffer.Length == 0) {
-            throw new InvalidOperationException("Cannot write because buffer length is 0");
-         }
-         var count = values.Count;
-         var nPages = count / buffer.Length;
-         using var en = values.GetEnumerator();
-         for (int pg = 0; pg < nPages; pg++) {
-            for (int i = 0; i < buffer.Length; i++) {
-               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
-               buffer[i] = en.Current;
-            }
-            writer.WriteBatch(buffer!);
-         }
-         var remainder = count - nPages * buffer.Length;
-         if (remainder != 0) {
-            for (int i = 0; i < remainder; i++) {
-               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
-               buffer[i] = en.Current;
-            }
-            writer.WriteBatch(buffer!, 0, remainder);
-         }
-      }
-
-      public void Dispose() =>
-         writer.Dispose();
-
-      void IColumnWriter.Write(IReadOnlyCollection<object?> values) =>
-         Write(ReadOnlySelectorCollection.Create(values, convert));
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetColumnReader.cs

@@ -1,111 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts;
-using bts.collections;
-
-using ParquetSharp;
-
-using System;
-using System.Collections.Generic;
-
-class ParquetColumnReader<T> : IParquetColumnReader<T> {
-   protected readonly LogicalColumnReader<T> reader;
-   protected readonly T?[] buffer;
-
-   public ParquetColumnReader(ColumnReader reader, int bufferSize)
-      : this(reader, new T?[bufferSize]) { }
-
-   protected ParquetColumnReader(ColumnReader reader, T?[] buffer) {
-      this.buffer = buffer;
-      this.reader = reader.LogicalReader<T>();
-   }
-
-   ~ParquetColumnReader() =>
-      Dispose();
-
-   public IList<T?> Read() {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      int n = reader.ReadBatch(buffer!);
-      return buffer.WrapSubRange(0, n);
-   }
-
-   public IList<T?> Read(int? start, int? stop) {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      start ??= 0;
-      stop ??= buffer.Length;
-      int length = stop.Value - start.Value;
-      if (length <= 0) {
-         return new List<T?>(0);
-      }
-      if (start.Value > 0) {
-         var nSkipped = checked((int)reader.Skip(start.Value));
-         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
-         if (nSkipped < start.Value) {
-            return new List<T?>(0);
-         }
-      }
-      int n = reader.ReadBatch(buffer!, 0, length);
-      return buffer.WrapSubRange(0, n);
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      reader.Dispose();
-   }
-}
-
-class ParquetColumnReader<TIn, TOut> : IParquetColumnReader<TOut> {
-   readonly Func<TIn, TOut?> convert;
-   readonly LogicalColumnReader<TIn> reader;
-   readonly TIn[] buffer;
-
-   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, int bufferSize)
-      : this(reader, convert, new TIn[bufferSize]) { }
-
-   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, TIn[] buffer) {
-      this.convert = convert;
-      this.reader = reader.LogicalReader<TIn>();
-      this.buffer = buffer;
-   }
-
-   ~ParquetColumnReader() =>
-      Dispose();
-
-   public IList<TOut?> Read() {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      int n = reader.ReadBatch(buffer!);
-      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
-   }
-
-   public IList<TOut?> Read(int? start, int? stop) {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      start ??= 0;
-      stop ??= buffer.Length;
-      int length = stop.Value - start.Value;
-      if (length <= 0) {
-         return new List<TOut?>(0);
-      }
-      if (start.Value > 0) {
-         var nSkipped = checked((int)reader.Skip(start.Value));
-         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
-         if (nSkipped < start.Value) {
-            return new List<TOut?>(0);
-         }
-      }
-      int n = reader.ReadBatch(buffer!, 0, length);
-      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      reader.Dispose();
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetExtensions.cs

@@ -1,159 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Microsoft.Scripting.Utils;
-
-using ParquetSharp;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-public static class ParquetExtensions {
-   public static int RowCount(this FileMetaData fileMetaData) =>
-      checked((int)fileMetaData.NumRows);
-
-   public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
-      var colInfos = new ColumnInfo[schemaDescriptor.NumColumns];
-      for (int i = 0; i < colInfos.Length; i++) {
-         colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata);
-      }
-      return new BlueTableSchema(tableName, colInfos);
-   }
-
-   public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") =>
-      ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata);
-
-   public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
-      var c = schemaDescriptor.Column(index);
-      var lt = c.LogicalType;
-      if (lt.Type == LogicalTypeEnum.Enum) {
-         var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name);
-         if (enumType is not null) {
-            return new(c.Name, index, enumType);
-         }
-      }
-      return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType));
-   }
-
-   public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) {
-      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      IBlueRowCollection? rv;
-      if (!schema.columnInfo.Any(c => c.colName == "id")) {
-         rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop));
-      } else {
-         rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop));
-      }
-      Log.assert(!schema.columnInfo.Any(c => c is null));
-      return rv;
-   }
-
-   public static (Column[] Column, IReadOnlyDictionary<string, string> Meta) ToParquetSchema(this IBlueTableSchema schema) {
-      var blueColumns = schema.columnInfo;
-      var parquetColumns = new Column[blueColumns.Length];
-      var parquetMeta = new Dictionary<string, string>();
-      KeyValuePair<string, string>[] parquetColMeta;
-      for (int i = 0; i < blueColumns.Length; i++) {
-         (parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]);
-         parquetMeta.AddRange(parquetColMeta);
-      }
-      return (parquetColumns, parquetMeta);
-   }
-
-   public static (Column Column, KeyValuePair<string, string>[] Meta) ToParquetColumn(this ColumnInfo columnInfo) {
-      var colType = columnInfo.colType;
-      bool isNullable = colType.IsNullable();
-      if (isNullable) {
-         colType = colType.GetGenericArguments()[0];
-      }
-      KeyValuePair<string, string>[] meta;
-      TypeCode typeCode;
-      var colName = columnInfo.colName;
-      if (colType.IsEnum) {
-         typeCode = colType.GetEnumUnderlyingType().GetTypeCode();
-         meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) };
-      } else {
-         typeCode = colType.GetTypeCode();
-         meta = Array.Empty<KeyValuePair<string, string>>();
-      }
-      return (ToParquetColumn(colName, typeCode), meta);
-
-      static Column ToParquetColumn(string colName, TypeCode typeCode) =>
-         typeCode switch {
-            TypeCode.Boolean => new Column<bool?>(colName),
-            TypeCode.Byte => new Column<byte?>(colName),
-            TypeCode.Int16 => new Column<short?>(colName),
-            TypeCode.Int32 => new Column<int?>(colName),
-            TypeCode.Int64 => new Column<long?>(colName),
-            TypeCode.UInt16 => new Column<ushort?>(colName),
-            TypeCode.UInt32 => new Column<uint?>(colName),
-            TypeCode.UInt64 => new Column<ulong?>(colName),
-            TypeCode.Single => new Column<float?>(colName),
-            TypeCode.Double => new Column<double?>(colName),
-            TypeCode.DateTime => new Column<DateTimeNanos?>(colName),
-            TypeCode.String => new Column<string?>(colName),
-            _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
-         };
-   }
-
-   static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) {
-      return logicalType.Type switch {
-         // LogicalTypeEnum.Int => TypeCode.Int32,
-         // LogicalTypeEnum.Decimal => TypeCode.Double,
-         LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime,
-         LogicalTypeEnum.String => TypeCode.String,
-         LogicalTypeEnum.Int => FromInt(),
-         LogicalTypeEnum.Decimal => FromDecimal(),
-         _ => FromPhysicalType(),
-      };
-
-      TypeCode FromPhysicalType() =>
-         physicalType switch {
-            PhysicalType.Boolean => TypeCode.Boolean,
-            PhysicalType.Int32 => TypeCode.Int32,
-            PhysicalType.Int64 => TypeCode.Int64,
-            PhysicalType.Float => TypeCode.Single,
-            PhysicalType.Double => TypeCode.Double,
-            _ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"),
-         };
-
-      TypeCode FromInt() {
-         var ilt = (IntLogicalType)logicalType;
-         var bitWidth = ilt.BitWidth;
-         return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt();
-
-         TypeCode FromSignedInt() =>
-            bitWidth switch {
-               8 => TypeCode.SByte,
-               16 => TypeCode.Int16,
-               32 => TypeCode.Int32,
-               64 => TypeCode.Int64,
-               _ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"),
-            };
-
-         TypeCode FromUnsignedInt() =>
-            bitWidth switch {
-               8 => TypeCode.Byte,
-               16 => TypeCode.UInt16,
-               32 => TypeCode.UInt32,
-               64 => TypeCode.UInt64,
-               _ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"),
-            };
-      }
-
-      TypeCode FromDecimal() {
-         var dlt = (DecimalLogicalType)logicalType;
-         var scale = dlt.Scale;
-         var precision = dlt.Precision;
-         return scale switch {
-            0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"),
-            _ => TypeCode.Double,
-         };
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetGroupTable.cs

@@ -1,234 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-public class ParquetGroupTable : IBlueRowCollection, IDisposable {
-   readonly RowGroupReader reader;
-   readonly IBlueTableSchema schema;
-   readonly bool leaveOpen;
-   readonly int fullCount;
-   readonly object syncRoot = new();
-   readonly int? start;
-   readonly int? stop;
-   readonly int startIndex;
-   readonly int stopIndex;
-   readonly int idColumnIndex;
-   readonly int idColumnIndexInFile;
-   readonly Func<int, int>? rowIdFactory;
-   DataCache? dataCacheBacking;
-
-   public ParquetGroupTable(
-      RowGroupReader reader,
-      IBlueTableSchema schema,
-      int? start = null,
-      int? stop = null,
-      int idColumnIndex = -1,
-      int idColumnIndexInFile = -1,
-      Func<int, int>? rowIdFactory = null,
-      bool leaveOpen = false
-   ) {
-      this.reader = reader;
-      this.schema = schema;
-      this.start = start;
-      this.stop = stop;
-      this.idColumnIndex = idColumnIndex;
-      this.idColumnIndexInFile = idColumnIndexInFile;
-      this.rowIdFactory = rowIdFactory;
-      this.leaveOpen = leaveOpen;
-      fullCount = checked((int)reader.MetaData.NumRows);
-      Log.assert(fullCount >= 0);
-      startIndex = SliceList.GetStartIndex(start, fullCount, true);
-      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
-      stopIndex = Math.Max(stopIndex, startIndex);
-      Log.assert(
-         startIndex >= 0
-         && startIndex <= fullCount
-         && stopIndex >= 0
-         && stopIndex <= fullCount
-         && stopIndex >= startIndex);
-   }
-
-   ~ParquetGroupTable() =>
-      Dispose();
-
-   public int Count => stopIndex - startIndex;
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   DataCache dataCache =>
-      EnsureInit();
-
-   public IBlueRow this[int id] => dataCache[id];
-
-   public bool containsID(int id) =>
-      dataCache.RowIdToIndex.ContainsKey(id);
-
-   public IBlueRow GetRow(int index) =>
-      dataCache.Rows[index];
-
-   public int IndexOf(IBlueRow item) {
-      int id = item.Id();
-      return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item)
-         ? idx
-         : -1;
-   }
-
-   public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) {
-      if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) {
-         row = dataCache.Rows[idx];
-         return true;
-      }
-      row = null;
-      return false;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      for (int i = 0; i < dataCache.Rows.Length; i++) {
-         yield return dataCache.Rows[i];
-      }
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      if (!leaveOpen) {
-         reader.Dispose();
-      }
-   }
-
-   [MemberNotNull(nameof(dataCacheBacking))]
-   internal DataCache EnsureInit() {
-      if (dataCacheBacking.HasValue) {
-         return dataCacheBacking.Value;
-      }
-      lock (syncRoot) {
-         if (dataCacheBacking.HasValue) {
-            return dataCacheBacking.Value;
-         }
-         var rows = new IBlueRow[Count];
-         int numColumns = reader.MetaData.NumColumns;
-         var blueColumns = schema.columnInfo;
-         Log.assert((blueColumns.Length - numColumns) switch {
-            0 or 1 => true,
-            _ => false,
-         });
-         var columnValues = allocateColumnValues(
-            reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory);
-         var rowIdToIndex = new Dictionary<int, int>();
-         for (int i = 0; i < rows.Length; i++) {
-            rows[i] = new BlueRowColumnWrapper(schema, i, columnValues);
-         }
-         dataCacheBacking = new(rows, rowIdToIndex, columnValues);
-      }
-      return dataCacheBacking.Value;
-   }
-
-   static IList<object?>[] allocateColumnValues(
-      RowGroupReader reader,
-      int numColumns,
-      int numRows,
-      int idColIdx,
-      int idColIdxInFile,
-      int startIndex,
-      int stopIndex,
-      Func<int, int>? idFactory
-   ) {
-      if (idColIdx < 0 || idColIdx >= numColumns) {
-         throw new ArgumentException("Id column index out of bounds", nameof(idColIdx));
-      }
-      var columnValues = new IList<object?>[numColumns];
-      if (idColIdxInFile == -1) {
-         // Generate ids (there is no id column in the file)
-         ArgumentNullException.ThrowIfNull(idFactory);
-         for (int i = 0; i < numColumns; i++) {
-            if (i < idColIdx) {
-               ReadColumn(i, i);
-            } else if (i == idColIdx) {
-               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
-            } else {
-               ReadColumn(i - 1, i);
-            }
-         }
-      } else if (idFactory is not null) {
-         // Generate ids (skip id column in the file)
-         for (int i = 0; i < numColumns; i++) {
-            if (i == idColIdx) {
-               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
-            } else {
-               if (i > idColIdx && i < idColIdxInFile) {
-                  ReadColumn(i - 1, i);
-               } else if (i > idColIdxInFile && i < idColIdx) {
-                  ReadColumn(i + 1, i);
-               } else {
-                  ReadColumn(i, i);
-               }
-            }
-         }
-      } else {
-         // Get ids from the file (output table id column index may differ from file)
-         // Case 1) File id col index > output
-         //    src cols: a, b, c, d, (id), e, f, ...
-         //    dst cols: a, b, (id), c, d, e, f, ...
-         //
-         // Case 2) File id col index == output
-         //    src cols: a, b, (id), c, d, e, f, ...
-         //    dst cols: a, b, (id), c, d, e, f, ...
-         //
-         // Case 3) File id col index < output
-         //    src cols: a, b, (id), c, d, e, f, ...
-         //    dst cols: a, b, c, d, (id), e, f, ...
-
-         for (int i = 0; i < numColumns; i++) {
-            if (i == idColIdx) {
-               ReadColumn(idColIdxInFile, idColIdx);
-            } else if (i > idColIdx && i < idColIdxInFile) {
-               ReadColumn(i - 1, i);
-            } else if (i > idColIdxInFile && i < idColIdx) {
-               ReadColumn(i, i - 1);
-            } else {
-               ReadColumn(i, i);
-            }
-         }
-      }
-      return columnValues;
-
-      void ReadColumn(int srcIdx, int dstIdx) {
-         using var colReader = reader.GetColumnReader(srcIdx);
-         columnValues[dstIdx] = colReader.Read(startIndex, stopIndex);
-      }
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-
-   internal readonly struct DataCache {
-      public readonly IBlueRow[] Rows;
-      public readonly Dictionary<int, int> RowIdToIndex;
-      public readonly IList<object?>[] ColumnValues;
-
-      public DataCache(IBlueRow[] rows, Dictionary<int, int> rowIdToIndex, IList<object?>[] columnValues) {
-         Rows = rows;
-         RowIdToIndex = rowIdToIndex;
-         ColumnValues = columnValues;
-      }
-
-      public IBlueRow this[int id] =>
-         Rows[RowIdToIndex[id]];
-
-      public IBlueRow GetRow(int index) =>
-         Rows[index];
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetReaderStream.cs

@@ -1,65 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Threading;
-
-[DataStreamUserRespresentation(false)]
-public class ParquetReaderStream : ViewStream {
-   readonly string file;
-   readonly int? start;
-   readonly int? stop;
-   readonly IWorkScheduler workScheduler;
-   IBlueRowCollection? table;
-   IDisposable? workerSub;
-
-   public ParquetReaderStream(string file, IWorkScheduler workScheduler) {
-      this.file = file;
-      this.workScheduler = workScheduler;
-   }
-
-   public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) {
-      this.file = file;
-      this.start = start;
-      this.stop = stop;
-      this.workScheduler = workScheduler;
-   }
-
-   protected override void myOnFirstSubscriber() {
-      Start();
-      var worker = new AnonymousWorker(DoWork);
-      workScheduler.Add(worker);
-      workerSub = Disposable.Create(() => workScheduler.Remove(worker));
-
-      async void Start() {
-         try {
-            Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop));
-         } catch (Exception ex) {
-            Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}");
-         }
-      }
-
-      void DoWork() {
-         var table = Volatile.Read(ref this.table);
-         if (table is null) {
-            return;
-         }
-         Disposable.dispose_thread_safe(ref workerSub);
-         if (table.schema.columnInfo.Length > 1) {
-            myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose_thread_safe(ref workerSub);
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetRow.cs

@@ -1,26 +0,0 @@
-// namespace Bts.DataStream.Parquet;
-
-// using BCG;
-
-// using global::Parquet.Data;
-
-// public class ParquetRow : IBlueRow {
-//    readonly int id;
-//    readonly int rowIndex;
-//    readonly DataColumn[] data;
-
-//    public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) {
-//       this.id = id;
-//       this.rowIndex = rowIndex;
-//       this.data = data;
-//       Schema = schema;
-//    }
-
-//    public object? this[int index] =>
-//       index == 0
-//          ? id
-//          : data[index - 1].Data.GetValue(rowIndex);
-
-//    public IBlueTableSchema Schema { get; }
-//    IBlueTableSchema IBlueRow.schema => Schema;
-// }

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTable.cs

@@ -1,145 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-
-public class ParquetTable : IBlueRowCollection, IDisposable {
-   readonly SortedList<int, ParquetGroupTable> groups = new();
-   readonly int fullCount;
-   readonly IBlueTableSchema schema;
-   readonly ParquetFileReader? reader;
-   readonly int startIndex;
-   readonly int stopIndex;
-
-   public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) {
-      schema = reader.FileMetaData.ToBlueSchema(tableName);
-      int idColIdx;
-      if (schema.TryResolveIndices("id", out int idColIdxInFile)) {
-         idColIdx = idColIdxInFile;
-      } else {
-         var origCols = schema.columnInfo;
-         var cols = new ColumnInfo[origCols.Length + 1];
-         idColIdx = 0;
-         cols[0] = new("id", 0, typeof(int));
-         for (int i = 0; i < origCols.Length; i++) {
-            var c = origCols[i];
-            cols[i + 1] = new(c.colName, i + 1, c.colType);
-         }
-         schema = new BlueTableSchema(cols, schema.tableName);
-      }
-      fullCount = checked((int)reader.FileMetaData.NumRows);
-      Log.assert(fullCount >= 0);
-      startIndex = SliceList.GetStartIndex(start, fullCount, true);
-      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
-      stopIndex = Math.Max(stopIndex, startIndex);
-      Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex);
-      int numRowGroups = reader.FileMetaData.NumRowGroups;
-      int rowGroupLastIdx = 0;
-      object syncRoot = new();
-      for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) {
-         int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null;
-         int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null;
-         int offset = rowGroupLastIdx;
-         var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId);
-         if (grp.Count == 0) {
-            continue;
-         }
-         rowGroupLastIdx += grp.Count;
-         groups.Add(rowGroupLastIdx, grp);
-
-         int GetAutoId(int index) =>
-            index + offset;
-      }
-      Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit());
-      if (!leaveOpen) {
-         this.reader = reader;
-      }
-   }
-
-   ~ParquetTable() =>
-      Dispose();
-
-   public IBlueTableSchema Schema => schema;
-
-   public int Count => stopIndex - startIndex;
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public IBlueRow this[int index] {
-      get {
-         foreach (var grp in groups.Values) {
-            if (grp.TryGetValue(index, out var r)) {
-               return r;
-            }
-         }
-         throw new KeyNotFoundException();
-      }
-   }
-
-   public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") =>
-      new(new(path), tableName, start, stop, false);
-
-   public IBlueRow GetRow(int index) {
-      if (index < 0 || index >= Count) {
-         throw new IndexOutOfRangeException();
-      }
-      int rowGroupIdx = groups.Keys.BinarySearch(index);
-      if (rowGroupIdx < 0) {
-         rowGroupIdx = ~rowGroupIdx;
-      }
-      Log.assert(rowGroupIdx <= groups.Count);
-      var grp = groups[rowGroupIdx];
-      int idxWithinGrp = index;
-      if (rowGroupIdx > 0) {
-         idxWithinGrp -= groups.Keys[rowGroupIdx - 1];
-      }
-      return grp[idxWithinGrp];
-   }
-
-   public bool containsID(int id) =>
-      groups.Values.Any(rgl => rgl.containsID(id));
-
-   public int IndexOf(IBlueRow item) {
-      int idx;
-      int grpFirstIdx = 0;
-      foreach (var (grpLastIdx, grp) in groups) {
-         idx = grp.IndexOf(item);
-         if (idx >= 0) {
-            return idx + grpFirstIdx;
-         }
-         grpFirstIdx = grpLastIdx;
-      }
-      return -1;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      foreach (var rgl in groups.Values) {
-         foreach (var r in rgl) {
-            yield return r;
-         }
-      }
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      groups.Values.DisposeAll();
-      reader?.Dispose();
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableReader.cs

@@ -1,32 +0,0 @@
-#nullable enable
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using System.IO;
-using System.Threading.Tasks;
-
-public class ParquetTableReader {
-   public static Task<IBlueRowCollection> ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) =>
-      Task.Run(() => Read(stream, leaveOpen, start, stop));
-
-   public static Task<IBlueRowCollection> ReadAsync(string path, int? start = null, int? stop = null) =>
-      Task.Run(() => Read(path, start, stop));
-
-   public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) {
-      using var fileReader = new ParquetFileReader(stream, leaveOpen);
-      var rv = fileReader.ReadBlueTable(start, stop);
-      rv.schema.Validate();
-      return rv;
-   }
-
-   public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) {
-      IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop);
-      rv.schema.Validate();
-      return rv;
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableWriter.cs

@@ -1,38 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using bts;
-
-using ParquetSharp;
-
-using System.IO;
-using System.Threading.Tasks;
-
-public static class ParquetTableWriter {
-   public static Task WriteAsync(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) =>
-      Task.Run(() => Write(blueRows, outStream, leaveOpen));
-
-   public static void Write(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) {
-      var (parquetColumns, parquetMeta) = blueRows.schema.ToParquetSchema();
-      using var schema = Column.CreateSchemaNode(parquetColumns);
-      using var writerProps = createWriterProperties();
-      using var writer = new ParquetFileWriter(outStream, schema, writerProps, keyValueMetadata: parquetMeta, leaveOpen: leaveOpen);
-      using var rowGroup = writer.AppendRowGroup();
-      for (int i = 0; i < blueRows.schema.columnInfo.Length; i++) {
-         var col = blueRows.schema.columnInfo[i];
-         Log.assert(col.colNo == i);
-         using var colWriter = ParquetAdapter.GetColumnWriter(col.type_code(), rowGroup.NextColumn());
-         colWriter.Write(blueRows.GetColumn(i));
-      }
-      writer.Close();
-   }
-
-   static WriterProperties createWriterProperties(Compression compression = Compression.Snappy) {
-      using var builder = new WriterPropertiesBuilder();
-      builder.Compression(compression);
-      return builder.Build();
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetWriterStream.cs

@@ -1,87 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts.collections;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-using Spark.Core.Threading;
-
-using System;
-using System.IO;
-
-[DataStreamUserRespresentation(false)]
-public class ParquetWriterStream : ViewStream {
-   readonly string file;
-   readonly ViewStream inputStream;
-   readonly IWorkScheduler workScheduler;
-   readonly AtomicGate gate = new();
-   IDataPacket? pendingOutput;
-   IDisposable? dispose;
-
-   public ParquetWriterStream(ViewStream inputStream, string file, IWorkScheduler workScheduler) {
-      this.inputStream = inputStream;
-      this.file = file;
-      this.workScheduler = workScheduler;
-   }
-
-   protected override void myOnFirstSubscriber() {
-      BlueTableImmutable.DataPacketBuilder? outputBuilder = null;
-      bool isFirstUpdate = false;
-      var worker = new AnonymousWorker(DoWork);
-      inputStream.onChange += OnInputStreamChange;
-      workScheduler.Add(worker);
-      dispose = Disposable.Create(Dispose);
-
-      void Dispose() {
-         workScheduler.Remove(worker);
-         inputStream.onChange -= OnInputStreamChange;
-      }
-
-      void OnInputStreamChange(object? dpObj) {
-         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dpObj);
-         using (gate.BeginTransaction()) {
-            outputBuilder ??= new BlueTableImmutable(dp.Schema).ToDataPacketBuilder();
-            foreach (IBlueRow r in dp.addedRows) {
-               outputBuilder.AddOrUpdate(r);
-            }
-            foreach (IBlueRow r in dp.deletedRows) {
-               outputBuilder.Remove(r.Id());
-            }
-         }
-      }
-
-      async void DoWork() {
-         using (gate.BeginTransaction()) {
-            if (outputBuilder is null) {
-               return;
-            }
-            if (pendingOutput is not null) {
-               myFireEvent(pendingOutput);
-               pendingOutput = null;
-            }
-            var dp = outputBuilder.ToDataPacket();
-            if (isFirstUpdate) {
-               isFirstUpdate = false;
-            } else if (!dp.HasChanges()) {
-               return;
-            }
-            {
-               using var outStream = File.OpenWrite(file);
-               await ParquetTableWriter.WriteAsync(dp.newRows, outStream);
-            }
-            // Fire the update at the beginning of the next doWork call so it happens on the
-            // 'work' thread.
-            pendingOutput = dp;
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose_thread_safe(ref dispose);
-}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,5 +1,8 @@
 namespace bts.BlueOrb {
    using BCG;
+
+   using Spark.Core.Diagnostics;
+
    using System;
    using System.Collections;
    using System.Collections.Generic;
@@ -54,7 +57,7 @@ namespace bts.BlueOrb {
 
       public bool IsReadOnly { get { return true; } }
 
-      public object this[object key] {
+      public object? this[object key] {
          get {
             if (key is string)
                return this[(string)key];
@@ -63,16 +66,15 @@ namespace bts.BlueOrb {
          set { throw new NotSupportedException("The IBlueRow is readonly"); }
       }
 
-      public object this[int index] {
+      public object? this[int index] {
          get {
             return this[schema.columnInfo[index].colName];
          }
       }
 
-      public object this[string col_name] {
+      public object? this[string col_name] {
          get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
                return _row[mapped_to_col];
             }
             return _row[col_name];
@@ -89,7 +91,8 @@ namespace bts.BlueOrb {
          }
       }
 
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Add(object key, object? value) =>
+         throw new NotSupportedException();
 
       public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
 
@@ -114,7 +117,7 @@ namespace bts.BlueOrb {
 
       private ViewStream _stream;
       private string _project_expression;
-      private ProjectData _project_data = null;
+      ProjectData? _project_data;
       //public ProjectData 
       public NewProjectStream(ViewStream vs, string project_expression) {
          _stream = vs;
@@ -128,9 +131,8 @@ namespace bts.BlueOrb {
          return _project_data.out_schema;
       }
 
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
+      public IBlueRow map(IBlueRow row) =>
+         new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
 
       // this is where most of the logic is, in terms of translating cols
       private static ProjectData _create_project_data(IBlueTableSchema input_schema,

src/DataStreamSuite/DataStream/BlueOrb/TestStream.cs

@@ -0,0 +1,58 @@
+namespace BlueOrb.Internal;
+
+using BCG;
+
+using System;
+using System.Linq;
+
+[DataStreamUserRespresentation("TestStream")]
+public class TestStream : ViewStream {
+   public TestStream() {
+      var rs = new BlueOrb.Operators.Core.LiteralStream("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc");
+      mySetRealSource(rs);
+   }
+}
+
+[DataStreamUserRespresentation("OpsStream")]
+public class OpsStream : ViewStream {
+   public OpsStream() {
+   }
+
+   protected override void myOnFirstSubscriber() {
+      IBlueTableSchema schema = new BlueTableSchema("Ops");
+      schema = schema.add_column(System.TypeCode.Int32, "id");
+      schema = schema.add_column(System.TypeCode.String, "op");
+      schema = schema.add_column(System.TypeCode.String, "namespace");
+      schema = schema.add_column(System.TypeCode.String, "assembly_name");
+      schema = schema.add_column(System.TypeCode.String, "constructor");
+      var loaded_streams = PrimitiveManager.instance.get_loaded_streams();
+      BlueTable rv = new BlueTable(schema);
+      foreach (var kv in loaded_streams.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase)) {
+         string sn = kv.Key;
+         Type t = kv.Value;
+         string ns = t.Namespace ?? string.Empty;
+         string assembly_name = t.Assembly.GetName().Name ?? string.Empty;
+         var constructors = t.GetConstructors()
+            .OrderBy(c => c.GetParameters().Length)
+            .ThenBy(c => c.ToString());
+         if (!constructors.Any()) {
+            IBlueRow r = new BlueRow(schema, rv.Count, sn, ns, assembly_name, "(no public constructor)");
+            rv.addRow(r);
+            continue;
+         }
+         foreach (var ctor in constructors) {
+            var parameters = ctor.GetParameters();
+            string constructor = $"{t.Name}({string.Join(", ", parameters.Select(p => p.ParameterType.Name))})";
+            IBlueRow r = new BlueRow(schema, rv.Count, sn, ns, assembly_name, constructor);
+            rv.addRow(r);
+         }
+      }
+      myFireEvent(new DataPacket(rv, rv, new BlueTable(schema)));
+   }
+
+   protected override void myOnLastSubscriber() {
+      base.myOnLastSubscriber();
+   }
+
+
+}

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -1,18 +1,7 @@
-using BCG;
-
-using bts.utils;
-
-using Bts.DataStreamCore;
-
-using mbts.utils;
-
-using Spark.Core.Collections;
-using Spark.Core.Threading;
-
 using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-using System.Linq;
+using System.Data.Entity.Core.Mapping;
+
+using BCG;
 
 namespace bts.BlueOrb {
    // Keeps adding changing rows to the stream, hence preserving the history
@@ -20,20 +9,17 @@ namespace bts.BlueOrb {
    // into a time series of prices/qty
    [DataStreamUserRespresentation("TimeSeriesStream")]
    public class TimeSeriesStream : ViewStream {
-      readonly ViewStream _stream;
-      readonly string timeColumn;
-      readonly AtomicGate gate = new();
-      CircularBufferTable? _table;
+      private CircularBufferTable _table;
+      private ViewStream _stream;
       private int _id = 0;
       private int _size = 0;
 
       public TimeSeriesStream(ViewStream stream)
          : this(stream, Int32.MaxValue) { }
 
-      public TimeSeriesStream(ViewStream stream, int size, string timeColumn = "ts_event") {
+      public TimeSeriesStream(ViewStream stream, int size) {
          _stream = stream;
          _size = size;
-         this.timeColumn = timeColumn;
       }
 
       protected override void myOnFirstSubscriber() {
@@ -45,37 +31,27 @@ namespace bts.BlueOrb {
 
 
       private void _onstream_change(object o) {
-         var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
+         DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
-         using var _ = gate.BeginTransaction();
          if (_table == null) {
-            var colInfos = dp.newRows.schema.columnInfo;
-            if (colInfos.TryGetFirst(c => c.colName == timeColumn, out var currTimeCol, out var currTimeColIdx)) {
-               colInfos = colInfos.ToArray();
-               colInfos[currTimeColIdx] = new(timeColumn, currTimeCol.colNo, typeof(DateTime));
-            } else {
-               colInfos = ArrayHelper.add(colInfos, new(timeColumn, colInfos.Length, typeof(DateTime)));
-            }
-            _table = new CircularBufferTable(new BlueTableSchema(colInfos, dp.newRows.schema.tableName), _size);
+            IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
+            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            _table = new CircularBufferTable(schema, _size);
          }
-         var addedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
-         var deletedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue(timeColumn, now);
-            int id = _id++;
-            ar.setValue("id", id);
-            _table = _table.Add(ar, out var oldRow);
-            addedRowsBd.AddOrUpdate(ar);
-            if (oldRow is not null) {
-               deletedRowsBd.AddOrUpdate(oldRow);
-            }
+            ar.setValue("timestamp", now);
+            ar.setValue("id", _id++);
+            _table.add_row(ar);
          }
-         var addedRows = addedRowsBd.ToImmutable();
-         var deletedRows = deletedRowsBd.ToImmutable();
-         var dp_out = new DataPacket(_table, addedRows, deletedRows);
+         var dp_out =  BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
          myFireEvent(dp_out);
       }
    }
+
+
+
 }
 
+
+

src/DataStreamSuite/DataStream/BlueOrb/TimeseriesIndex.cs

@@ -1,197 +0,0 @@
-namespace Bts.DataStream;
-
-using bts;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-
-/// <summary>
-/// Maps a timeseries of keys to unique int32 ids.
-/// </summary>
-public class TimeseriesIndex<T> where T : notnull {
-   static readonly IComparer<KeyValuePair<DateTime, int>> idTimeseriesComp =
-      KeyValuePairComparer.KeyComparer<DateTime, int>();
-
-   readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId;
-   readonly ImmutableDictionary<int, TimeseriesKey<T>> idToName;
-   readonly int idCounter;
-
-   public TimeseriesIndex()
-      : this(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Empty, ImmutableDictionary<int, TimeseriesKey<T>>.Empty, 0) { }
-
-   TimeseriesIndex(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
-      this.nameToId = nameToId;
-      this.idToName = idToName;
-      this.idCounter = idCounter;
-   }
-
-   public int Count => idToName.Count;
-
-   public ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> NameToId => nameToId;
-   public ImmutableDictionary<int, TimeseriesKey<T>> IdToName => idToName;
-
-   public bool ContainsId(int id) =>
-      idToName.ContainsKey(id);
-
-   public int GetNearest(TimeseriesKey<T> key) {
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         throw new KeyNotFoundException();
-      }
-      int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-      if (idx < 0) {
-         idx = ~idx;
-         if (idx == 0) {
-            throw new KeyNotFoundException();
-         }
-         idx--;
-      }
-      return ids[idx].Value;
-   }
-
-   public bool TryGetNearest(TimeseriesKey<T> key, out int result) {
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         result = -1;
-         return false;
-      }
-      int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-      if (idx < 0) {
-         idx = ~idx;
-         if (idx == 0) {
-            result = -1;
-            return false;
-         }
-         idx--;
-      }
-      result = ids[idx].Value;
-      return true;
-   }
-
-   public TimeseriesIndex<T> GetNearestOrAdd(TimeseriesKey<T> key, out int result) {
-      int idx;
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-         idx = 0;
-      } else {
-         idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-         if (idx < 0) {
-            idx = ~idx;
-         }
-      }
-      if (idx == 0) {
-         result = idCounter + 1;
-         ids = ids.Insert(0, new(key.Timestamp, result));
-         return new(
-            nameToId.SetItem(key.Name, ids),
-            idToName.SetItem(result, key),
-            result);
-      }
-      result = ids[idx].Value;
-      return this;
-   }
-
-   public TimeseriesIndex<T> GetOrAdd(TimeseriesKey<T> key, out int result) {
-      int idx;
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-         idx = 0;
-      } else {
-         idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-         if (idx < 0) {
-            idx = ~idx;
-         } else {
-            result = ids[idx].Value;
-            return this;
-         }
-      }
-      result = idCounter + 1;
-      ids = ids.Insert(idx, new(key.Timestamp, result));
-      return new(
-         nameToId.SetItem(key.Name, ids),
-         idToName.SetItem(result, key),
-         result);
-   }
-
-   public IList<KeyValuePair<DateTime, int>> GetIds(T name) =>
-      nameToId[name];
-
-   public Builder ToBuilder() =>
-      new(this);
-
-   public class Builder {
-      readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Builder nameToId;
-      readonly ImmutableDictionary<int, TimeseriesKey<T>>.Builder idToName;
-      int idCounter;
-
-      public Builder(TimeseriesIndex<T> source)
-         : this(source.nameToId, source.idToName, source.idCounter) { }
-
-      public Builder(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
-         this.nameToId = nameToId.ToBuilder();
-         this.idToName = idToName.ToBuilder();
-         this.idCounter = idCounter;
-      }
-
-      public int GetNearest(TimeseriesKey<T> key) =>
-         getOrAdd(key, false, false);
-
-      public int GetNearestOrAdd(TimeseriesKey<T> key) =>
-         getOrAdd(key, false, true);
-
-      public int GetOrAdd(TimeseriesKey<T> key) =>
-         getOrAdd(key, true, true);
-
-      public int GetLatest(T key) =>
-         nameToId.TryGetValue(key, out var ids)
-            ? ids[^1].Value
-            : throw new KeyNotFoundException();
-
-      public bool TryGetLatest(T key, out int result) {
-         if (nameToId.TryGetValue(key, out var ids)) {
-            result = ids[^1].Value;
-            return true;
-         }
-         result = 0;
-         return false;
-      }
-
-      public ImmutableList<KeyValuePair<DateTime, int>> GetIds(T name) =>
-         nameToId.TryGetValue(name, out var ids)
-            ? ids
-            : ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-
-      public TimeseriesIndex<T> ToImmutable() =>
-         new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter);
-
-      int getOrAdd(TimeseriesKey<T> key, bool requireExactTimestamp, bool canAdd) {
-         var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList<KeyValuePair<DateTime, int>>.Empty);
-         int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp);
-         int id;
-         if (idx < 0) {
-            Log.Gen.info($"Adding key to timeseries index ({key.Timestamp}, {key.Name})");
-            // We didn't get an exact match. If requireExactTimestamp is false and a record with an
-            // earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp.
-            idx = ~idx;
-            if (requireExactTimestamp || idx == 0) {
-               if (!canAdd) {
-                  throw new KeyNotFoundException();
-               }
-               id = ++idCounter;
-               ids = ids.Insert(idx, new(key.Timestamp, id));
-               nameToId[key.Name] = ids;
-               idToName.Add(id, key);
-            } else {
-               if (idx == ids.Count) {
-                  --idx;
-               }
-               id = ids[idx].Value;
-            }
-         } else {
-            id = ids[idx].Value;
-         }
-         return id;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TimeseriesKey.cs

@@ -1,22 +0,0 @@
-namespace Bts.DataStream;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-
-public readonly record struct TimeseriesKey<T>(T Name, DateTime Timestamp) : IDualComparable<TimeseriesKey<T>> {
-   public int CompareTo(TimeseriesKey<T> other) {
-      var rv = Timestamp.CompareTo(other.Timestamp);
-      if (rv == 0) {
-         rv = Comparer<T>.Default.Compare(Name, other.Name);
-      }
-      return rv;
-   }
-
-   public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey<T> key) =>
-      (key.Name, key.Timestamp);
-
-   public static implicit operator TimeseriesKey<T>((T Name, DateTime Timestamp) tup) =>
-      new(tup.Name, tup.Timestamp);
-}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -5,7 +5,8 @@ namespace bts.BlueOrb {
    using mbts;
    using BCG;
 
-   [DataStreamUserRespresentation("UnionStream")]
+   // [DataStreamUserRespresentation("UnionStream")]
+   [DataStreamUserRespresentation("AppendStream")]
    public class UnionStream : ViewStream {
       public UnionStream(ViewStream left_stream, ViewStream right_stream) {
          ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -6,24 +6,37 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
-    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="ParquetSharp" Version="20.0.0" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+        <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
+        <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+        <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+        <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
+        <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ArithmeticExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="BaseUtilities" Version="$(SparkVersion)" />
+        <PackageReference Include="BooleanExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.ColumnExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.ColumnSortExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="DataStreamExpressions" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
+
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,7 +2,6 @@ namespace Bts.DataStream;
 
 using BCG;
 
-using bts;
 using bts.api.core;
 
 using Bts.DataStreamCore;
@@ -16,13 +15,27 @@ using Spark.Core.Threading;
 using System;
 using System.Linq;
 using System.Reactive.Linq;
-using System.Threading;
-using System.Threading.Tasks;
 
 public static class DataStreamExtensions {
+   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
+      source as ViewStream ?? new ObservableViewStream(source);
+
    public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
       source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
 
+   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<IDataPacket> observer) {
+         source.onChange += OnChange;
+         return Disposable.Create(() => source.onChange -= OnChange);
+
+         void OnChange(object? newValue) =>
+            observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
+               $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
+      }
+   }
+
    // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
    //    source.Select(dp => {
    //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
@@ -46,16 +59,16 @@ public static class DataStreamExtensions {
    public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
       this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
       rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
-         var bd = res.to_builder();
+      return source.Scan(TableUpdate<T, TKey>.Empty(keySelector), (res, dp) => {
+         var bd = res.ToBuilder();
          dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
          foreach (IBlueRow r in addedRows) {
-            bd.add_or_update(valueSelector(r));
+            bd.AddOrUpdate(valueSelector(r));
          }
          foreach (IBlueRow r in deletedRows) {
-            bd.remove(rowKeySelector(r));
+            bd.Remove(rowKeySelector(r));
          }
-         return bd.to_table();
+         return bd.ToImmutable();
       });
    }
 
@@ -65,16 +78,16 @@ public static class DataStreamExtensions {
          (res, table) => {
             var resTable = res.Table;
             if (resTable is null) {
-               if (table.state.Count == 0) {
+               if (table.Index.Count == 0) {
                   return (null, null);
                }
-               resTable = new(table.state.Values.First().schema);
+               resTable = new(table.Index.Values.First().schema);
             }
             var bd = resTable.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
+            foreach (var r in table.RemovedItems) {
                bd.Remove(r.Id());
             }
-            foreach (var r in table.added) {
+            foreach (var r in table.AddedItems) {
                bd.AddOrUpdate(r);
             }
             var resDp = bd.ToDataPacket(out var newRows);
@@ -88,10 +101,10 @@ public static class DataStreamExtensions {
          (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
          (res, table) => {
             var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
+            foreach (var r in table.RemovedItems) {
                bd.Remove(r.Id());
             }
-            foreach (var r in table.added) {
+            foreach (var r in table.AddedItems) {
                bd.AddOrUpdate(r);
             }
             var resDp = bd.ToDataPacket(out var newRows);
@@ -121,6 +134,46 @@ public static class DataStreamExtensions {
    public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
       source.AsDataPackets(schema).AsViewStream();
 
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
+      var columns = schema.columnInfo;
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
+         }
+      }
+      return indices;
+   }
+
    public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
       Observable.Create<IDataPacket>(o => {
          var inputGate = new AtomicGate();
@@ -145,7 +198,7 @@ public static class DataStreamExtensions {
                foreach (IBlueRow r in value.addedRows) {
                   bd.AddOrUpdate(r);
                }
-               pendingDpOut = bd.ToDataPacket(out lastTable, false);
+               pendingDpOut = bd.ToDataPacket(out lastTable);
             }
          }
 
@@ -161,118 +214,5 @@ public static class DataStreamExtensions {
          }
       });
 
-   public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
-      const int STATE_IDLE = 0;
-      const int STATE_BUSY = 1;
-      const int STATE_DISPOSED = 2;
-      return Observable.Create<IDataPacket>(Subscribe);
-      IDisposable Subscribe(IObserver<IDataPacket> o) {
-         var syncRoot = new object();
-         IBlueRowCollection? lastTableIn = null;
-         BlueTableImmutable? lastTableOut = null;
-         int nConcurrentDoWorks = 0;
-         int state = STATE_IDLE;
-         IDisposable? dispose = null;
-         dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
-         return Disposable.Create(Dispose);
 
-         void Dispose() {
-            lock (syncRoot) {
-               switch (state) {
-                  case STATE_DISPOSED:
-                     return;
-                  case STATE_BUSY:
-                     state = STATE_DISPOSED;
-                     return;
-               }
-               state = STATE_DISPOSED;
-            }
-            Disposable.dispose(ref dispose);
-         }
-
-         void OnNext(IDataPacket value) =>
-            Volatile.Write(ref lastTableIn, value.newRows);
-
-         async void DoWork() {
-            lock (syncRoot) {
-               nConcurrentDoWorks++;
-               switch (state) {
-                  case STATE_DISPOSED:
-                  case STATE_BUSY:
-                     return;
-                  case STATE_IDLE:
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
-                     return;
-               }
-               state = STATE_BUSY;
-            }
-            var dp = await PublishTable();
-            bool isDisposed = false;
-            lock (syncRoot) {
-               switch (state) {
-                  case STATE_DISPOSED:
-                     isDisposed = true;
-                     break;
-                  case STATE_BUSY:
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
-                     return;
-               }
-            }
-            if (isDisposed) {
-               Disposable.dispose(ref dispose);
-               return;
-            }
-            if (dp is not null) {
-               o.OnNext(dp);
-            }
-            lock (syncRoot) {
-               nConcurrentDoWorks = 0;
-               switch (state) {
-                  case STATE_DISPOSED:
-                     isDisposed = true;
-                     break;
-                  case STATE_BUSY:
-                     state = STATE_IDLE;
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
-                     return;
-               }
-            }
-            if (isDisposed) {
-               Disposable.dispose(ref dispose);
-            }
-
-            async Task<DataPacket?> PublishTable() {
-               var tableIn = Interlocked.Exchange(ref lastTableIn, null);
-               if (tableIn is null) {
-                  return null;
-               }
-               var tableOut = await transformAsync(tableIn);
-               if (tableOut is null) {
-                  return null;
-               }
-               lastTableOut ??= new(tableOut.schema);
-               var bd = lastTableOut.ToDataPacketBuilder();
-               bd.Clear();
-               foreach (IBlueRow r in tableOut) {
-                  bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
-               }
-               return bd.ToDataPacket(out lastTableOut);
-            }
-         }
-
-         static string StateName(int state) =>
-            state switch {
-               STATE_IDLE => "IDLE",
-               STATE_BUSY => "BUSY",
-               STATE_DISPOSED => "DISPOSED",
-               _ => $"STATE_{state}",
-            };
-      }
-   }
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -3,27 +3,30 @@ namespace bts.BlueOrb {
    using BCG;
    using System.Collections.Generic;
    using Bts.DataStreamExpressions;
-   using Bts.DataStreamCore;
+   using System.Xml;
 
    // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
    // So for example, if an expresison is passed 
    [DataStreamUserRespresentation("dsl")]
    public class DSLStream : ViewStream {
       private readonly string _dsl;
-      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
+      public DSLStream(string dsl) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         // Remove all colons
+         string sanitizedDsl = dsl.Replace(":", "COLON");
          _dsl = dsl;
-         var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
          mySetRealSource(rs);
       }
 
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
+      static ViewStream? build_stream(string stream_expression) {
          try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
          } catch (Exception ex) {
+            //send to blueorb
             Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,14 +1,13 @@
 namespace Bts.DataStream;
 
 using BCG;
-using BCG.Extensions;
 using BCG.Utilities.General;
 
 using bts;
-using bts.collections;
 
 using Bts.DataStreamCore;
 
+using Spark.Core.Collections;
 using Spark.Core.Disposables;
 
 using System;

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -40,7 +40,7 @@ namespace BCG {
         protected override void myOnFirstSubscriber() {
             MasterWorker.getInstance(false).addWorker(this, _send_period);
             var server_rci = new RemoteConnectionInfo(_ip, _port);
-            bts.AsyncTCPConnector_AsyncWrapper.connect_to(server_rci, this);
+            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
         }
 
         protected override void myOnLastSubscriber() {
@@ -76,4 +76,4 @@ namespace BCG {
 
 
     }
-}
+}

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -1,5 +1,4 @@
-#nullable enable
-namespace Bts.DataStreamCore;
+namespace Bts.DataStream;
 
 using BCG;
 

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -1,5 +1,6 @@
 using bts;
 
+using Bts.DataStream;
 using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
@@ -11,7 +12,6 @@ using Spark.Core.Disposables;
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Reactive.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -35,12 +35,12 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
+      static ViewStream? build_stream(string stream_expression)
       {
          try
          {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
          } catch (Exception ex) {
@@ -52,85 +52,43 @@ namespace BCG {
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       readonly IPubTableBroker _broker;
       readonly IManagedSubscriber _sub;
-      readonly IDataStreamFactory streamFactory;
-      readonly ITableStreamFactory tableStreamFactory;
       readonly object syncRoot = new();
       IDisposable? cloudStreamsSub;
       IDisposable? remoteRequestsSub;
-      State state;
+      bool isDisposed;
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, ITableStreamFactory tableStreamFactory, IEnumerable<IDataStreamFactory> streamFactories) {
+
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
          _sub = managedSubscriber;
          _broker = pubTableBroker;
-         this.tableStreamFactory = tableStreamFactory;
-         var streamFactoryList = streamFactories.ToList();
-         streamFactory = streamFactoryList.Count == 1
-            ? streamFactoryList[0]
-            : new CompositeDataStreamFactory(streamFactoryList);
-      }
-
-      enum State {
-         Idle = 0,
-         Running = 1,
-         Disposed = 2,
-         WaitingToDispose = 3,
       }
 
       public Task StartAsync(CancellationToken cancellationToken) {
          lock (syncRoot) {
-            switch (state) {
-               case State.Idle:
-                  break;
-               case State.WaitingToDispose:
-               case State.Disposed:
-                  throw new ObjectDisposedException(nameof(RemoteQueryManager));
-               default:
-                  return Task.CompletedTask;
-            }
-            state = State.Running;
-         }
-         if (cloudStreamsSub is not null) {
-            throw new InvalidOperationException("cloudStreamsSub already created");
-         }
-         if (remoteRequestsSub is not null) {
-            throw new InvalidOperationException("remoteRequestsSub already created");
-         }
-         var cloudStream = tableStreamFactory.Create("cloud_streams")
-            ?? throw new Exception("Failed to create TableStream 'cloud_stream'");
-         cloudStreamsSub = cloudStream.AsObservable().Subscribe(_on_str);
-         remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
-         lock (syncRoot) {
-            if (state == State.WaitingToDispose) {
-               state = State.Disposed;
-            }
-         }
-         if (state == State.Disposed) {
-            Disposable.dispose(ref cloudStreamsSub);
-            Disposable.dispose(ref remoteRequestsSub);
+            Log.assert(!isDisposed);
+            Log.isnull(cloudStreamsSub);
+            Log.isnull(remoteRequestsSub);
+            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
+            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
+               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+            });
          }
          return Task.CompletedTask;
       }
 
       public Task StopAsync(CancellationToken cancellationToken) {
          lock (syncRoot) {
-            switch (state) {
-               case State.Running:
-                  break;
-               default:
-                  return Task.CompletedTask;
+            if (!isDisposed) {
+               isDisposed = true;
+               Disposable.dispose(ref cloudStreamsSub);
+               Disposable.dispose(ref remoteRequestsSub);
             }
-            state = State.Disposed;
          }
-         Disposable.dispose(ref cloudStreamsSub);
-         Disposable.dispose(ref remoteRequestsSub);
          return Task.CompletedTask;
       }
 
-      private void _on_str(IDataPacket dp)
-      {
+      void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
@@ -144,11 +102,7 @@ namespace BCG {
 
       private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         var stream = build_stream(streamFactory, stream_expression);
-         if (stream is null) {
-            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
-            return;
-         }
+         ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
          if (persist) {
             string tempDir = Path.GetTempPath();

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -2,30 +2,17 @@ using BCG;
 
 using bts;
 
-using Bts.DataStreamCore;
-
-using Spark.Core.Disposables;
-
 using System;
 
 namespace BCG {
-   /// <summary>
-   /// This is a stream so we can run it from the DSQA (since dsqa can only load streams).
-   /// <para>
-   /// Note: It's hidden because this stream is now created by an IDataStreamFactory
-   /// (see RemoteQueryStreamFactory.cs for details)
-   /// </para>
-   /// </summary>
-   [DataStreamUserRespresentation(false)]
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
    public class RemoteQueryStream : ViewStream {
       private readonly string _name, _stream_exp;
-      readonly ITableStreamFactory tableStreamFactory;
-      readonly IPublisherCore _publisher_core;
-      IDisposable? subscription;
+      private PublisherCore _publisher_core = PublisherCore.Create();
+      private ViewStream _source_str;
 
-      public RemoteQueryStream(ITableStreamFactory tableStreamFactory, IPublisherCore publisherCore, params string[] lines) {
-         this.tableStreamFactory = tableStreamFactory;
-         _publisher_core = publisherCore;
+      public RemoteQueryStream(params string[] lines) {
          _stream_exp = get_expression(lines);
          _name = Guid.NewGuid().ToString();
          _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
@@ -34,17 +21,17 @@ namespace BCG {
       }
 
       protected override void myOnFirstSubscriber() {
-         Log.isnull(subscription);
          RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
          byte[] rem_req_bytes = rem_req.serialize();
          _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
-         var sourceStream = tableStreamFactory.Create(_name);
-         sourceStream.onChange += _handle_data;
-         subscription = Disposable.Create(() => sourceStream.onChange -= _handle_data);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
       }
 
-      protected override void myOnLastSubscriber() =>
-         Disposable.dispose(ref subscription);
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }
 
       static string get_expression(params string[] lines) {
          for (int i = 0; i < lines.Length; i++) {

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -2,9 +2,6 @@ namespace bts {
    using System.IO;
    using BCG;
 
-   using System.IO;
-   using System.Threading.Tasks.Dataflow;
-
    public class RemoteRequest {
       public readonly string stream_expression;
       public readonly string published_table_name;
@@ -28,4 +25,4 @@ namespace bts {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,33 +1,28 @@
+using Spark.Core.Collections;
+
 using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace BCG {   
    public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_single_tok(string exp) {
+      public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
          string[] toks = exp.Split(':');
          string col_name = toks[0].Trim();
          string col_value = toks[1].Trim();
          Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return Tuple.Create(col_name, tup.Item1, tup.Item2);
+         return (col_name, tup.Item1, tup.Item2);
       }
 
-      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
-         string[] toks = exp.Split(",");
-         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
-         foreach (string s in toks) {
-            var tup = parse_single_tok(s.Trim());
-            rv.Add(tup);
-         }
-         return rv;
-      }
+      public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
+         exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
 
-      private Tuple<string, Type, object> _new_col;
-      private List<Tuple<string, Type, object>> _new_col_map;
-      private IBlueTableSchema _out_schema;
+      readonly List<(string Name, Type Type, object Value)> _new_col_map;
+      IBlueTableSchema? _out_schema;
 
       public AddColumnsStream(ViewStream vs, string col_def) {
          _new_col_map = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
+         ViewStream rs = new MapStream(vs, _get_schema, _map);
          mySetRealSource(rs);
       }
 
@@ -44,7 +39,7 @@ namespace BCG {
       private IBlueRow _map(IBlueRow r) {
          MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
          foreach (var m in _new_col_map) {
-            rv.setValue(m.Item1, m.Item3);
+            rv.setValue(m.Name, m.Value);
          }
          return rv;
       }

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs

@@ -1,9 +1,10 @@
-using System;
+using mbts;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using mbts;
 
 namespace BCG
 {
@@ -33,7 +34,7 @@ namespace BCG
 
          private void _stream_onChange(object newValue)
          {
-            DataPacket dp = newValue as DataPacket;
+            var dp = (IDataPacket)newValue;
             if (dp != null && dp.newRows.Count > 0)
             {
                _stream.onChange -= _stream_onChange;
@@ -42,8 +43,8 @@ namespace BCG
          }
       }
 
-      public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
-      public delegate string GetColumnPrefix(object[] transposing_col_values);
+      public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
+      public delegate string GetColumnPrefix(object?[] transposing_col_values);
 
       public delegate string GetColumnSuffix(string col_name);
       private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
@@ -51,33 +52,31 @@ namespace BCG
       private IncrementValues _increment_values;
       private GetColumnPrefix _get_column_prefix;
 
-      private IBlueTableSchema _schema;
+      IBlueTableSchema? _schema;
 
       private string[] _group_by_cols;
       private Pair<string, Type>[] _cols_per_transposed;
       private string[] _transposing_cols;
 
-      private int[] _transposing_col_indices;
+      int[]? _transposing_col_indices;
 
-      private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
+      Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
          : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
                 get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
       { }
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
       {
          if (get_column_suffix_del != null)
             _get_column_suffix = get_column_suffix_del;
          if (transposed_col_value_comparer != null)
          {
-            _transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
-               {
-                  return transposed_col_value_comparer(p1.second, p2.second);
-               });
+            _transpsed_col_value_comparer_wrapper = new((p1, p2) =>
+               transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
          }
          _group_by_cols = group_by_cols.Split(' ');
          _transposing_cols = transposed_cols_group.Split(' ');
@@ -106,7 +105,7 @@ namespace BCG
             col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
          foreach (string tc_prefix in tc_prefixes)
             foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
          IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
          if (_schema == null)
             _schema = schema;
@@ -130,19 +129,14 @@ namespace BCG
 
       public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
       {
-         Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
-         Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
+         var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
+         var group_by_cols_value_dict = new Dictionary<int, object?[]>();
          BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
          Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
          string table_name = string.Empty;
          if (grouped_rows.Count > 0)
          {
-            IBlueTableSchema schema = null;
-            foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
-            {
-               schema = kvp.Value[0].schema;
-               break;
-            }
+            var schema = grouped_rows.Values.First()[0].schema;
             table_name = schema.tableName;
             _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
             int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
@@ -155,23 +149,22 @@ namespace BCG
          else
             return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); 
          Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
-         List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
+         var tc_values = new List<Pair<int, object?[]>>();
          foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
          {
             //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values 
-            Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
+            var incrementing_values = new Dictionary<int, object?[]>();
             foreach (IBlueRow r in kvp.Value)
             {
                int hash = hasher.get_hash(r, _transposing_col_indices);
-               object[] values;
-               if (!incrementing_values.TryGetValue(hash, out values))
+               if (!incrementing_values.TryGetValue(hash, out var values))
                {
                   incrementing_values.Add(hash, _get_per_transposing_col_values(r));
                   string prefix = _get_column_prefix(_get_transposing_col_values(r));
                   if (!tc_prefixes.ContainsKey(hash))
                   {
                      tc_prefixes.Add(hash, prefix);
-                     tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
+                     tc_values.Add(new(hash, _get_transposing_col_values(r)));
                   }
                }
                else
@@ -194,18 +187,18 @@ namespace BCG
          _set_schema(prefixes, group_by_col_info, table_name);
          BlueTable result = new BlueTable(_schema);
          int id = 1;
-         foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
+         foreach (var kvp in all_tc_infos_and_values)
          {
             MutableBlueRow mbr = new MutableBlueRow(_schema);
             mbr.setValue("id", id++);
             for (int i = 0; i < _group_by_cols.Length; i++)
                mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
-            foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
+            foreach (var tc_cols in kvp.Value)
             {
                string prefix = tc_prefixes[tc_cols.Key];
                for (int i = 0; i < _cols_per_transposed.Length; i++)
                {
-                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
+                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
                   if(mbr.schema.columnNames.Contains(column))
                      mbr.setValue(column, tc_cols.Value[i]);
                }
@@ -217,28 +210,25 @@ namespace BCG
 
       #endregion
 
-      private object[] _get_transposing_col_values(IBlueRow row)
-      {
-         object[] values = new object[_transposing_cols.Length];
+      object?[] _get_transposing_col_values(IBlueRow row) {
+         var values = new object?[_transposing_cols.Length];
          for (int i = 0; i < _transposing_cols.Length; i++)
             values[i] = row[_transposing_cols[i]];
          return values;
       }
 
-      private object[] _get_per_transposing_col_values(IBlueRow row)
-      {
-         object[] values = new object[_cols_per_transposed.Length];
+      object?[] _get_per_transposing_col_values(IBlueRow row) {
+         var values = new object?[_cols_per_transposed.Length];
          for (int i = 0; i < _cols_per_transposed.Length; i++)
-            values[i] = row[_cols_per_transposed[i].first];
+            values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
          return values;
       }
 
-      private object[] _get_group_by_cols_values(IBlueRow row)
-      {
-         object[] values = new object[_group_by_cols.Length];
+      object?[] _get_group_by_cols_values(IBlueRow row) {
+         var values = new object?[_group_by_cols.Length];
          for (int i = 0; i < values.Length; i++)
             values[i] = row[_group_by_cols[i]];
          return values;
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -2,7 +2,7 @@ using System;
 
 namespace bts.BlueOrb {
    using BCG;
-   [DataStreamUserRespresentation("oldCount")]
+   [DataStreamUserRespresentation("CountStream")]
    public class CountStream : ViewStream {
       private DataPacket myLastFiredDataPacket;
       private IBlueTableSchema mySchema = new BlueTableSchema("count",
@@ -15,6 +15,9 @@ namespace bts.BlueOrb {
          myVS = vs;
       }
 
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
+
       protected override void myOnFirstSubscriber() {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -1,22 +1,19 @@
-using System;
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 
 namespace BCG
 {
    public class DerivativeStream : ViewStream 
    {
-
-      private IBlueTableSchema mySchema;
+      IBlueTableSchema? mySchema;
       private string myDerivandColumn;
       private string myOutputColumn;
       private string myKeyColumn;
       private DateTime myLastPacketArrival = DateTime.MinValue;      
       private ViewStream myDataSource;
-      
-      private IDictionary<string, double> myPreviousValues;
-
+      Dictionary<string, double>? myPreviousValues;
 
       public DerivativeStream(ViewStream input, string keyColumn,  string columnToDifferentiate, string outputColName)
       {
@@ -39,9 +36,9 @@ namespace BCG
 
       void myDataSource_onChange(object newValue)
       {
-         DataPacket dp = (DataPacket)newValue;
+         var dp = (IDataPacket)newValue;
          if (myLastPacketArrival == DateTime.MinValue)
-            mySchema = myBuildOutputSchema(dp.schema);
+            mySchema = myBuildOutputSchema(dp.Schema);
          else
          {
             DateTime now = DateTime.Now;
@@ -55,7 +52,7 @@ namespace BCG
             else
             {
                IBlueRowCollection deleted, added;
-               BlueDataUtility.computeDifferences(thisVersion, ((DataPacket)myLastValue).newRows, out deleted, out added);
+               BlueDataUtility.computeDifferences(thisVersion, ((IDataPacket)myLastValue).newRows, out deleted, out added);
                myFireEvent(new DataPacket(thisVersion, added, deleted));  
             }
          }
@@ -64,12 +61,13 @@ namespace BCG
 
       protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
       {
+         ExceptionAssert.NotNull(mySchema, myPreviousValues);
          object[] outValues = new object[mySchema.columnInfo.Length];
          r.Values.CopyTo(outValues, 0);
 
-         object derivand = r[myDerivandColumn];
-         double lastValue = default(double);
-         if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value)
+         var derivand = r[myDerivandColumn];
+         var key = getRowKey(r);
+         if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
             outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
          else
             outValues[mySchema.columnInfo.Length - 1] = 0.0;
@@ -84,16 +82,19 @@ namespace BCG
          cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double));
          return new BlueTableSchema("derviative of " + inSchema.tableName, cols);
       }
-      protected void myStoreNewPacketInfo(DataPacket dp)
-      {
+
+      protected void myStoreNewPacketInfo(IDataPacket dp) {
          myPreviousValues = new Dictionary<string, double>();
          myLastPacketArrival = DateTime.Now;
          foreach (IBlueRow row in dp.newRows)
          {
-            object derivand = row[myDerivandColumn];
+            var derivand = row[myDerivandColumn];
             if (derivand != null && derivand != DBNull.Value)
-               myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand);
+               myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
          }
       }
+
+      string getRowKey(IBlueRow row) =>
+         row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
    }
 }

src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs

@@ -1,7 +1,9 @@
-using System;
-using System.Collections;
+using mbts;
+
+using Spark.Core.Collections;
+
+using System;
 using System.Collections.Generic;
-using mbts;
 
 namespace BCG
 {
@@ -12,9 +14,9 @@ namespace BCG
    {
       ViewStream myLStream, myRStream;
 
-      DataPacket myLastR = null, myLastL = null;
-
-      BlueTable myLastFired = null;
+      DataPacket? myLastR;
+      DataPacket? myLastL;
+      BlueTable? myLastFired;
 
       double myThreshold = .00005;
 
@@ -24,14 +26,14 @@ namespace BCG
       public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : 
          this(null, v1, v2, thresh, ColumnsToMatch) { }
 
-      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
       {
          myThreshold = thresh;
          myLStream = v1;
          myRStream = v2;
          myColumnsToMatch = columnsToMatch;
       }
-      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
          : this(db,v1,v2,.00005,columnsToMatch)
       {
       }
@@ -50,45 +52,22 @@ namespace BCG
          myRStream.onChange -= myRChange;
       }
 
-      struct ListComparer : IEqualityComparer<List<object>>
-      {
-         public bool Equals(List<object> x, List<object> y)
-         {
-            if (x.Count != y.Count)
-               return false;
-
-            for (int i = 0; i < x.Count; ++i)
-               if (!x[i].Equals(y[i]))
-                  return false;
-            return true;
-         }
-
-         public int GetHashCode(List<object> obj)
-         {
-            int ret = 0;
-            for(int i = 0; i < obj.Count; ++i)
-               ret ^= obj[i].GetHashCode();
-            return ret;
-         }
-      }
-
       void myHandleCallbacks()
       {
-         DataPacket packet = null;
-
+         DataPacket? packet;
          if (myLastL == null || myLastR == null)
          {
             packet = myLastL == null ? myLastR : myLastL;
-            if (packet != null)
+            if (packet is not null) {
                myFireEvent(packet);
-
-            myLastFired = new BlueTable(packet.newRows);
+               myLastFired = new BlueTable(packet.newRows);
+            }
             return;
          }
 
          BlueTable table = new BlueTable(myLastL.newRows.schema);
 
-         Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
+         var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
 
          foreach(BlueRow row in myLastL.newRows)
          {
@@ -118,11 +97,7 @@ namespace BCG
                }
                catch (Exception) { }
             }
-
-            Pair<BlueRow, BlueRow> pair;
-            rowPairs.TryGetValue(hash, out pair);
-
-
+            rowPairs.TryGetValue(hash, out var pair);
             if (pair == null)
                pair = new Pair<BlueRow, BlueRow>(null, row);
             else
@@ -138,8 +113,7 @@ namespace BCG
 
             if (pair.first == null || pair.second == null)
             {
-               BlueRow row = pair.first == null ? pair.second : pair.first;
-
+               var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
                Dictionary<string, object> newRow = new Dictionary<string, object>();
                foreach (ColumnInfo info in row.schema.columnInfo)
                {
@@ -211,4 +185,4 @@ namespace BCG
 
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs

@@ -14,10 +14,8 @@ namespace BCG
 		public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
          : base (colMapExpression, null, columnNames) {}
 
-
-      public override IBlueRow evaluate (DataPacket packet)
-      {
-         myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
+      public override IBlueRow? evaluate(IDataPacket packet) {
+         myOutputTable ??= new GroupByOutputTable(packet.Schema);
          string hashString = GroupByStream.getRepString (packet, myGroupByCols);
          if (packet.newRows.Count == 0)
             //if there are no rows for this group we remove the group from the output table

src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs

@@ -5,7 +5,7 @@ namespace BCG
 {
 	public class FileViewStream : ViewStream
 	{
-      private DataPacket myDp;
+      DataPacket? myDp;
 		public FileViewStream (string dataFileName, 
          string schemaFileName)
          : this (dataFileName, schemaFileName, Int32.MaxValue)

src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs

@@ -1,8 +1,7 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
+
 namespace BCG
 {
     public class DateBasedFilter: IDynamicStreamFilter, IWorker
@@ -26,20 +25,14 @@ namespace BCG
             myReference = myTodaysDate.AddDays(-1 * myNDays);                     
         }       
         
-        public event FilterChangeDelegate onFilterChange;       
+        public event FilterChangeDelegate? onFilterChange;       
         
-        public bool filter(IBlueRow r)
-        {
-            if (DBNull.Value.Equals(r[myDateColumn]))
-            {
-                //return false;
-                return true; //it can be considered as our assumption  whenever datecolumn is db null we return true 
-            }
-            DateTime t = (DateTime)r[myDateColumn];                   
-            bool returnVallue =  t < myReference ? false : true;
-            return returnVallue;
-        }
-              
+        public bool filter(IBlueRow r) =>
+           r[myDateColumn] switch {
+              DBNull or null => true,
+              DateTime t => t >= myReference,
+              _ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
+           };
 
         public void doWork()
         {
@@ -47,7 +40,7 @@ namespace BCG
             {
                 myTodaysDate = DateTime.Today;
                 myReference = myTodaysDate.AddDays(-1 * myNDays);
-                onFilterChange();
+                onFilterChange?.Invoke();
             }         
         }      
     }

src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs

@@ -1,8 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
 
 namespace BCG
 {
@@ -26,7 +24,7 @@ namespace BCG
 
       #region IDynamicStreamFilter Members
 
-      public event FilterChangeDelegate onFilterChange;
+      public event FilterChangeDelegate? onFilterChange;
 
       #endregion
 
@@ -34,7 +32,10 @@ namespace BCG
 
       public bool filter(IBlueRow r)
       {
-         DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date;
+         DateTime rowDate = r[myDateColName] switch {
+            DBNull or null => DateTime.Now.Date,
+            object obj => (DateTime)obj,
+         };
          DateTime today = DateTime.Now.Date;
          return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
       }
@@ -48,7 +49,7 @@ namespace BCG
          if (DateTime.Now.Date != myLastCheckedTime.Date)
          {
             myLastCheckedTime = DateTime.Now;
-            onFilterChange();
+            onFilterChange?.Invoke();
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,18 +1,23 @@
 using bts;
 
-using Bts.DataStream;
 using Bts.DataStream.Primitives.Filter;
+using Bts.DataStreamCore;
 
-using mbts.config;
+using Spark.Core.Diagnostics;
+using Spark.Core.Disposables;
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
 using System.Text.RegularExpressions;
 
 namespace BCG.StreamWorld.Addition {
    [DataStreamUserRespresentation(false)]
    public class FilterStream : ViewStream {
       private ViewStream myBaseStream;
-      private BlueTable myBaseTable, myStateTable;
+      BlueTable? myBaseTable;
+      BlueTable? myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
@@ -25,8 +30,8 @@ namespace BCG.StreamWorld.Addition {
          myBaseStream = vs;
          myFilter = filter;
          vs.ErrorChanged += OnError;
-         if (filter is IDynamicStreamFilter)
-            (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         if (filter is IDynamicStreamFilter dfilter)
+            dfilter.onFilterChange += myHandleFilterChange;
       }
 
 
@@ -36,8 +41,8 @@ namespace BCG.StreamWorld.Addition {
 
       protected override void myOnLastSubscriber() {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
-         myBaseTable.clear();
-         myStateTable.clear();
+         myBaseTable?.clear();
+         myStateTable?.clear();
       }
 
       bool filterRow(IBlueRow r) {
@@ -52,17 +57,14 @@ namespace BCG.StreamWorld.Addition {
 
       private void myHandleChanges(object newValue) {
          try {
-            DataPacket dp = newValue as DataPacket;
-            if (myStateTable == null) {
-               bts.Log.assert(myBaseTable == null);
-               myStateTable = new BlueTable(dp.schema);
-               myBaseTable = new BlueTable(dp.schema);
-            }
-            BlueTable dels = new BlueTable(dp.schema),
-                        adds = new BlueTable(dp.schema);
+            var dp = (IDataPacket)newValue;
+            var schema = dp.Schema;
+            init(schema);
+            var dels = new BlueTable(schema);
+            var adds = new BlueTable(schema);
             foreach (IBlueRow r in dp.deletedRows) {
                myBaseTable.deleteRow(r);
-               if (myStateTable.containsID((int)r["id"])) {
+               if (myStateTable.containsID(r.Id())) {
                   dels.addRow(r);
                   myStateTable.deleteRow(r);
                }
@@ -85,6 +87,7 @@ namespace BCG.StreamWorld.Addition {
          set {
             myFilter = value;
             if (myBaseTable != null) {
+               ExceptionAssert.NotNull(myStateTable);
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
                // If the filter throws an exception, we can rethrow it as a StreamFilterException
@@ -112,6 +115,17 @@ namespace BCG.StreamWorld.Addition {
       //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
 
       private void myHandleFilterChange() { filter = myFilter; }
+
+      [MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
+      void init(IBlueTableSchema schema) {
+         if (myStateTable is null) {
+            Log.isnull(myBaseTable);
+            myStateTable = new BlueTable(schema);
+            myBaseTable = new BlueTable(schema);
+         } else {
+            Log.notnull(myBaseTable);
+         }
+      }
    }
 }
 
@@ -135,29 +149,24 @@ namespace BCG
       }
 
       #region IStreamFilter Members
-      public bool filter(IBlueRow dr)
-      {
-         bool rv = true;
-         for (int i = 0; i < myCols.Length; i++)
-         {
-            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-            if (!rv)
-               return rv;
-         }
-         return rv;
-      }
+      public bool filter(IBlueRow dr) =>
+         myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
+            DBNull or null => string.Empty,
+            object obj => obj.ToString() ?? string.Empty
+         }));
       #endregion
    }
 
    public class FilterStream : ViewStream
    {
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
-      private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
+      BlueTable? myLastFiredRows;
+      IBlueRowCollection? myLastReceivedRows;
+      IStreamFilter? myFilter;
+      readonly ViewStream? myParentStream;
+      string? myFilterString;
+      readonly StreamWorld.Addition.FilterStream? myFilterStream;
+      readonly bool myUseNewFilter = false;
+      IDisposable? subscription;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -187,9 +196,10 @@ namespace BCG
          {
             myParentStream = vs;
             myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+            myFilterString = filter.ToString() ?? string.Empty;
+            if (filter is IDynamicStreamFilter dfilter) {
+               dfilter.onFilterChange += myHandleFilterChange;
+            }
          }
       }
 
@@ -213,18 +223,20 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber()
-      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() {
+         ExceptionAssert.IsNull(subscription);
+         subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
+      }
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         Disposable.Dispose(ref subscription);
       }
 
       protected override void myFireEvent(object o)
       {
-         DataPacket dp = o as DataPacket;
+         var dp = (IDataPacket)o;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -233,29 +245,30 @@ namespace BCG
       private void myHandleCallbacks(object newValue)
       {
          DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
+         var arrivedPacket = (IDataPacket)newValue;
          myLastReceivedRows = arrivedPacket.newRows;
          try {
             if (myLastFiredRows != null)
             {
-               retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-                  new BlueTable(myLastFiredRows.schema));
+               var addedRows = new BlueTable(myLastFiredRows.schema);
+               var deletedRows = new BlueTable(myLastFiredRows.schema);
+               retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
                foreach (IBlueRow r in arrivedPacket.deletedRows)
-                  if (myLastFiredRows.containsID((int)r["id"]))
+                  if (myLastFiredRows.containsID(r.Id()))
                   {
                      myLastFiredRows.deleteRow(r);
-                     (retVal.deletedRows as BlueTable).addRow(r);
+                     deletedRows.addRow(r);
                   }
                foreach (IBlueRow r in arrivedPacket.addedRows)
                   if (filterRow(r))
                   {
                      myLastFiredRows.addRow(r);
-                     (retVal.addedRows as BlueTable).addRow(r);
+                     addedRows.addRow(r);
                   }
             }
             else
             {
-               myLastFiredRows = new BlueTable(arrivedPacket.schema);
+               myLastFiredRows = new BlueTable(arrivedPacket.Schema);
                retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
                foreach (IBlueRow r in arrivedPacket.newRows)
                   if (filterRow(r))
@@ -273,7 +286,7 @@ namespace BCG
          {
             if (myUseNewFilter)
             {
-               myFilterStream.filter = filter;
+               requireNewFilter().filter = filter;
             }
             else
             {
@@ -302,9 +315,9 @@ namespace BCG
          get
          {
             if (myUseNewFilter)
-               return myFilterStream.filter;
+               return requireNewFilter().filter;
             else
-               return myFilter;
+               return requireOldFilter();
          }
       }
 
@@ -317,17 +330,25 @@ namespace BCG
 
       private void myHandleFilterChange()
       {
-         filter = myFilter;
+         filter = requireOldFilter();
       }
 
       bool filterRow(IBlueRow r) {
          try {
-            return myFilter.filter(r);
+            return requireOldFilter().filter(r);
          } catch (Exception ex) {
             Log.Gen.error($"FilterStream delegate threw exception: {ex}");
             OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
          }
          return false;
       }
+
+      [MemberNotNull(nameof(myFilterStream))]
+      StreamWorld.Addition.FilterStream requireNewFilter() =>
+         ExceptionAssert.MemberNotNull(myFilterStream);
+
+      [MemberNotNull(nameof(myFilter))]
+      IStreamFilter requireOldFilter() =>
+         ExceptionAssert.MemberNotNull(myFilter);
    }
 }

src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs

@@ -1,8 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
 
 namespace BCG
 {    
@@ -29,7 +27,7 @@ namespace BCG
             myReference = myCurrentTime.AddSeconds(-1 * myNSecs);                  
         }
         
-        public event FilterChangeDelegate onFilterChange;       
+        public event FilterChangeDelegate? onFilterChange;       
         
         public bool filter(IBlueRow r)
         {
@@ -38,9 +36,8 @@ namespace BCG
                 //it can be considered as our assumption  whenever datecolumn is db null we return true 
                 return true;  //dont use this for volparamsched use true              
             }
-            DateTime t = (DateTime)r[myDateColumn];                   
-            bool returnVallue =  t < myReference ? false : true;
-            return returnVallue;
+            var t = (DateTime)r[myDateColumn]!;
+            return t >= myReference;
         }
               
 
@@ -51,8 +48,7 @@ namespace BCG
                 myCurrentTime = DateTime.Now;
                 myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
                 myLastTimeFilteredApplied = DateTime.Now;              
-                onFilterChange();
-                
+                onFilterChange?.Invoke();
             }           
         }
     }

src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs

@@ -1,11 +1,13 @@
-using System;
+using BCG.Utilities.General;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
 
 using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
-using System.Diagnostics;
 
 namespace BCG
 {
@@ -40,12 +42,12 @@ namespace BCG
       private IComparer<IBlueRow> myRightCompare;
       protected FuzzyRowComparer myFuzzyCompare;
       protected TimeSpan myTimeWindow;
-      protected IBlueTableSchema myLeftSchema;
-      protected IBlueTableSchema myRightSchema;
-      protected IBlueTableSchema myOutputSchema;
+      protected IBlueTableSchema? myLeftSchema;
+      protected IBlueTableSchema? myRightSchema;
+      protected IBlueTableSchema? myOutputSchema;
       private int myID = 0;
 
-      protected TimeWindowTable myStateRows;
+      protected TimeWindowTable? myStateRows;
 
       private DateTime myNextCull;
       const long MinCullIntervalMS = 1000;
@@ -65,7 +67,7 @@ namespace BCG
          this.myFuzzyCompare = fuzzyCompare;
       }
 
-      public virtual IBlueTableSchema OutputSchema
+      public virtual IBlueTableSchema? OutputSchema
       {
          get
          {
@@ -150,7 +152,7 @@ namespace BCG
          int count;
          for (count = 0; count < list.Count; count++)
          {
-            if ((DateTime)list[count][timeColumn] > expiry)
+            if ((DateTime)list[count][timeColumn]! > expiry)
             {
                // Stop when we find the first row that we can keep.  Since this is sorted all others are also kept.
                break;
@@ -172,16 +174,16 @@ namespace BCG
             this.myFitness = fitness;
          }
          public IBlueRow myRightRow { get; private set; }
-         public IBlueRow myLeftRow { get; set; }
+         public IBlueRow? myLeftRow { get; set; }
          public double myFitness { get; private set; }
-         public IBlueRow myMergedRow { get; set; }
+         public IBlueRow? myMergedRow { get; set; }
 
          #region IComparable<RowMatch> Members
 
-         public int CompareTo(RowMatch other)
-         {
-            return myFitness.CompareTo(other.myFitness);
-         }
+         public int CompareTo(RowMatch? other) =>
+            other is null
+               ? 1
+               : myFitness.CompareTo(other.myFitness);
 
          #endregion
       }
@@ -262,12 +264,12 @@ namespace BCG
 
       private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
       {
+         ExceptionAssert.NotNull(myStateRows);
          // We already emitted correction.MergedRow so we need to redact that and 
          // then emit correction.left as an add since it is no longer matched
          foreach (RowMatch correction in corrections.Distinct())
          {
-            Debug.Assert(correction.myMergedRow != null);
-            this.myStateRows.RemoveRow(correction.myMergedRow);
+            myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow));
             adds.addRow(this.MergeRows(correction.myLeftRow, null));
             dels.addRow(correction.myMergedRow);
          }
@@ -285,7 +287,7 @@ namespace BCG
          int r = 0;
          for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
          {
-            if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
+            if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!)
             {
                // Advance Left
                IEnumerable<RowMatch> rm = rightMatches[l];
@@ -332,7 +334,7 @@ namespace BCG
 
       private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
       {
-         IBlueRow merged = null;
+         IBlueRow? merged = null;
          if (rightMatches != null)
          {
             foreach (RowMatch match in rightMatches)
@@ -366,9 +368,9 @@ namespace BCG
       /// Override this function to provide a custom row merge.
       /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
       /// </summary>
-      protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
+      protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
       {
-         Dictionary<string, object> newRow = new Dictionary<string, object>();
+         Dictionary<string, object?> newRow = new();
 
          if (left == null && right == null) throw new ArgumentException("oh come on.");
 
@@ -393,14 +395,15 @@ namespace BCG
             }
          }
          newRow["id"] = this.myID++;
-         newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
-
+         newRow[NEW_TIME_COLUMN] = left is not null
+            ? left[myLeftTimeColumn]
+            : right![myRightTimeColumn];
          return new BlueRow(this.OutputSchema, newRow);
       }
 
       private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
       {
-         List<RowMatch> matches = null;
+         List<RowMatch>? matches = null;
 
          // Search through all rows in list and compare with the fuzzy comparer
          for (int x = 0; x < list.Count; x++)
@@ -421,8 +424,7 @@ namespace BCG
                matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
 
                // Mark the best match for this particular right hand match
-               RowMatch oldMatch;
-               if (this.myBestMatch.TryGetValue(match, out oldMatch))
+               if (this.myBestMatch.TryGetValue(match, out var oldMatch))
                {
                   if (currentMatch.myFitness < oldMatch.myFitness)
                   {
@@ -441,7 +443,7 @@ namespace BCG
             }
          }
 
-         return matches;
+         return matches ?? Enumerable.Empty<RowMatch>();
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -42,7 +42,7 @@ namespace BCG
          return retVal;
       }
 
-      public abstract IBlueRow evaluate (DataPacket dp) ;
+      public abstract IBlueRow? evaluate(IDataPacket dp);
       #endregion
 
       #region Helper methods
@@ -69,14 +69,7 @@ namespace BCG
       protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
       #endregion
 
-      private IBlueRow myProjectRow (IBlueRow r, string colMapExpression)
-      {
-         BlueTable auxTable = new BlueTable (r.schema);
-         auxTable.addRow (r);
-         BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable);
-         ProjectStream projStream = new ProjectStream (staticStream, colMapExpression);
-         return BlueDataUtility.extractRow (projStream.rows, 0);
-      }
-
+      IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
+         new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project"));
 	}
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -8,12 +8,13 @@ namespace BCG {
    internal class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
-         private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
+         readonly Func<IDataPacket, IBlueRow> _groupby_func;
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
             _groupby_func = groupby_func;
          }
-         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
+         public IBlueRow evaluate(IDataPacket packet) =>
+            _groupby_func(packet);
 
          public string[] groupByColumns { get { return _groupby_cols; } }
 
@@ -32,7 +33,7 @@ namespace BCG {
       private IBlueTableSchema myOutputSchema;
       private int myIdGenerator = 0;
 
-      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
+      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
          : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
       public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
@@ -187,11 +188,11 @@ namespace BCG {
          }
       }
 
-      public static string getRepString(DataPacket dp, string[] colNames) {
+      public static string getRepString(IDataPacket dp, string[] colNames) {
          return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow(DataPacket dp) {
+      public static IBlueRow getRepRow(IDataPacket dp) {
          if (dp.newRows.Count > 0)
             return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)

src/DataStreamSuite/DataStream/Primitives/GroupBy/IGroupByEvaluator.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG
 {
    /// <summary>
@@ -14,7 +12,7 @@ namespace BCG
       /// deletetion) and the added and deletedRows being the increments. If a group has
       /// to be totally removed then a null should be returned
       /// </summary>
-      IBlueRow evaluate (DataPacket packet);
+      IBlueRow? evaluate(IDataPacket packet);
 
       /// <summary>
       /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -14,9 +14,8 @@ namespace BCG {
         public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMaxRow = myOutputTable.getRow(hashString);
             IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,7 +1,3 @@
-using System;
-
-using BCG.Utilities;
-
 namespace BCG {
     public class MinGroupByEvaluator : MaxGroupByEvaluator {
         public MinGroupByEvaluator(string outputCol, string[] groupByCols)
@@ -10,8 +6,8 @@ namespace BCG {
         public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
            base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMinRow = myOutputTable.getRow(stringRep);
             IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);