refactor: Add new glbx variable ticks to security (shm and proto)

This makes it consistent with other type names (we don't use "Sparkd"
anywhere else).

.devcontainer/devcontainer.json

@@ -1,5 +1,11 @@
 {
-  "dockerFile": "Dockerfile",
+  "build": {
+    "dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
+    "options": [
+      "--secret",
+      "id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
+    ]
+  },
   "hostRequirements": {
     "memory": "8gb",
     "storage": "10gb"

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.vscode/launch.json

@@ -142,14 +142,14 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "preLaunchTask": "Build DataStream (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
           "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -164,8 +164,8 @@
         "program": "DataStreamQueryAnalyzer.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
           "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -188,9 +188,6 @@
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
       ],
       "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -212,6 +209,43 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "Sdk Demo",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "preLaunchTask": "Build Api Server (Debug)",
+      "windows": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "linux": {
+        "program": "Bts.Spark.Sdk.Demo",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sdk Demo (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "linux": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
     {
       "name": "Spark",
       "type": "coreclr",
@@ -300,6 +334,7 @@
       "program": "",
       "linux": {
         "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "preLaunchTask": "Build Sparkd (Debug | Clang)",
       },
       "args": [
         "run"
@@ -349,6 +384,25 @@
         }
       ]
     },
+    {
+      "name": "Sparkd Attach",
+      "type": "cppdbg",
+      "request": "attach",
+      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
   ],
   "inputs": [
     {
@@ -374,6 +428,16 @@
       ],
       "default": "MarketDataPublisher",
     },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
@@ -395,9 +459,9 @@
       "default": "PubSubServer",
     },
     {
-      "id": "pubSubServerEnvironment",
+      "id": "sdkDemoEnvironment",
       "type": "pickString",
-      "description": "Api Server Configuration",
+      "description": "Sdk Demo Configuration",
       "options": [
         "Development",
         "Local",
@@ -406,14 +470,10 @@
       "default": "Development",
     },
     {
-      "id": "dataStreamQueryAnalyzerEnvironment",
-      "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
+      "id": "sparkConfigServer",
+      "type": "promptString",
+      "description": "Config Server URL",
+      "default": "http://localhost:8000/config",
     },
     {
       "id": "sparkEnvironment",
@@ -425,12 +485,6 @@
       ],
       "default": "Development",
     },
-    {
-      "id": "sparkConfigServer",
-      "type": "promptString",
-      "description": "Config Server URL",
-      "default": "http://localhost:8000/config",
-    },
     {
       "id": "sparkTest",
       "type": "promptString",

.vscode/tasks.json

@@ -121,57 +121,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Build DataStreamQueryAnalyzer",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Build Spark",
       "group": "build",
@@ -223,6 +172,66 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Build Spark Sdk",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark Sdk (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark Sdk (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Build Sparkd",
       "group": "build",
@@ -527,17 +536,16 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStream",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -547,17 +555,16 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -567,17 +574,16 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStream (Release)",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -1039,6 +1045,16 @@
       ],
       "default": "Debug",
     },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkdCmakePreset",
       "type": "pickString",
@@ -1081,7 +1097,6 @@
       "options": [
         "Development",
         "EchoServer",
-        "Local",
         "Production",
         "ProductionAws",
         "Staging",

Bts.Spark.Sdk.sln

@@ -0,0 +1,162 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{3BF4DF96-5AFF-4674-866A-71A8090627FF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{62067659-AA2E-4CB5-B7CE-0B38EAA92080}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.Build.0 = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.Build.0 = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.Build.0 = Release|x64
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.Build.0 = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.Build.0 = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.Build.0 = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.ActiveCfg = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.Build.0 = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.ActiveCfg = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.Build.0 = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.Build.0 = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.Build.0 = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.ActiveCfg = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.Build.0 = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.Build.0 = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.Build.0 = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.Build.0 = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.ActiveCfg = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.Build.0 = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.ActiveCfg = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.Build.0 = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.Build.0 = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.Build.0 = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.ActiveCfg = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+	EndGlobalSection
+EndGlobal

CMakePresets.json

@@ -8,14 +8,31 @@
 
   "configurePresets": [
     {
-      "name": "spark",
+      "name": "spark-base",
+      "hidden": true,
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    },
+
+    {
+      "name": "spark-windows",
+      "inherits": "spark-base",
+      "displayName": "Spark",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
@@ -23,18 +40,9 @@
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
       }
     },
 
@@ -151,17 +159,17 @@
     },
 
     {
-      "name": "spark-api-server-windows",
-      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)"
-    },
-
-    {
-      "name": "spark-api-server-linux",
-      "inherits": "sparkd",
-      "displayName": "Spark Api Server (Linux)",
+      "name": "spark-linux",
+      "inherits": "spark-base",
+      "displayName": "Spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
+        "platform": "linux",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -170,21 +178,26 @@
           "type": "BOOL",
           "value": "OFF"
         },
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
         "CMAKE_EXPORT_COMPILE_COMMANDS": {
           "type": "BOOL",
           "value": "ON"
         },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "EXPORT_MD_CONSUMER": {
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
+        "DATABENTO_USE_EXTERNAL_JSON": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "EXTERNALS_ROOT": "${sourceDir}/externals/linux"
       }
     },
 
@@ -204,87 +217,32 @@
       "cacheVariables": {
         "CCAPI_LOG_LEVEL": "Trace"
       }
-    },
-
-    {
-      "name": "data-stream-suite-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
-      "inherits": "spark",
-      "displayName": "Data Stream Suite (Windows)",
-      "description": "Data Stream utils ported over from Edge",
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
-      }
-    },
-
-    {
-      "name": "data-stream-suite-linux",
-      "displayName": "Data Stream Suite (Linux)",
-      "description": "Data Stream utils ported over from Edge",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
-      "inherits": "sparkd",
-      "generator": "Ninja Multi-Config",
-      "cacheVariables": {
-        "BUILD_ADMIN_WEB_APP": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SDK": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SPARKD": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_C_COMPILER": "clang",
-        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
-      }
     }
   ],
 
   "buildPresets": [
     {
-      "name": "spark",
+      "name": "spark-windows",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
-      "hidden": true
+      "hidden": true,
+      "configurePreset": "spark-windows"
     },
 
     {
-      "name": "spark-debug",
+      "name": "spark-windows-debug",
       "displayName": "Spark (Debug)",
-      "inherits": "spark",
-      "configurePreset": "spark",
+      "inherits": "spark-windows",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-release",
+      "name": "spark-windows-release",
       "displayName": "Spark (Release)",
-      "inherits": "spark",
-      "configurePreset": "spark",
+      "inherits": "spark-windows",
       "configuration": "Release"
     },
 
@@ -327,52 +285,27 @@
     },
 
     {
-      "name": "spark-api-server-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
-      "hidden": true,
-      "configurePreset": "spark-api-server-windows"
-    },
-
-    {
-      "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Windows | Debug)",
-      "inherits": "spark-api-server-windows",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Windows | Release)",
-      "inherits": "spark-api-server-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "spark-api-server-linux",
+      "name": "spark-linux",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
       "hidden": true,
-      "configurePreset": "spark-api-server-linux"
+      "configurePreset": "spark-linux"
     },
 
     {
-      "name": "spark-api-server-linux-debug",
-      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Debug)",
+      "name": "spark-linux-debug",
+      "inherits": "spark-linux",
+      "displayName": "Spark (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-api-server-linux-release",
-      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Release)",
+      "name": "spark-linux-release",
+      "inherits": "spark-linux",
+      "displayName": "Spark (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -389,34 +322,6 @@
       "configurePreset": "sparkd-ccapi-diag",
       "displayName": "Debug (ccapi diag)",
       "inherits": "sparkd"
-    },
-
-    {
-      "name": "data-stream-suite-windows-debug",
-      "displayName": "Data Stream Suite (Windows | Debug)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-windows-release",
-      "displayName": "Data Stream Suite (Windows | Release)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "data-stream-suite-linux-debug",
-      "displayName": "Data Stream Suite (Linux | Debug)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-linux-release",
-      "displayName": "Data Stream Suite (Linux | Release)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Release"
     }
   ]
 }

Spark.Api.Server.sln

@@ -1,10 +1,11 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
@@ -42,180 +43,642 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sdk", "sdk", "{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\sdk\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}"
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{802ECF0B-7264-4168-95D1-158E47FB65CC}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{0354655D-2394-42BB-B1D4-2D75B9BCABF1}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
+		MinSizeRel|Any CPU = MinSizeRel|Any CPU
+		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
+		Release|Any CPU = Release|Any CPU
+		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
+		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
+		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.Build.0 = Debug|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.ActiveCfg = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.Build.0 = Release|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.Build.0 = Debug|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.ActiveCfg = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.Build.0 = Release|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.Build.0 = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.Build.0 = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.Build.0 = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.Build.0 = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.Build.0 = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.Build.0 = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.Build.0 = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.Build.0 = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.Build.0 = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.Build.0 = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.Build.0 = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.Build.0 = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.Build.0 = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.Build.0 = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.Build.0 = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.Build.0 = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.Build.0 = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.Build.0 = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.Build.0 = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.Build.0 = Release|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.Build.0 = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.Build.0 = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.ActiveCfg = Release|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.Build.0 = Release|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.ActiveCfg = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.Build.0 = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.ActiveCfg = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.Build.0 = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.ActiveCfg = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.Build.0 = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.ActiveCfg = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.Build.0 = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.ActiveCfg = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.Build.0 = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.ActiveCfg = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.Build.0 = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.ActiveCfg = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.Build.0 = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.ActiveCfg = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.Build.0 = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
@@ -227,10 +690,21 @@ Global
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184} = {56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{802ECF0B-7264-4168-95D1-158E47FB65CC} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
 	EndGlobalSection
 EndGlobal

cmake/Protobuf.cmake

@@ -1,182 +0,0 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
-set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
-
-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
-      DEPENDS ${ABS_FIL}
-      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
-
-    set(TEMPLATES
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
-    )
-
-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
-      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
-        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
-        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE}
-          ARGS
-            --cpp_out=${_ARGS_CXX_OUT_DIR}
-            --plugin=protoc-gen-service=${_plugin_cmd}
-            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
-            -I ${PROTOBUF_INCLUDE_DIRS}
-            -I ${_protobuf_src_dir}
-            ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_cmd}
-          ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
-    else()
-      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
-        ARGS
-          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
-    endif()
-  endforeach()
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-

compose.yaml

@@ -16,8 +16,8 @@ services:
         if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
           mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
+        if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
           set -a
           . /home/bts/.local/share/bts/sparkd/conf/.env.local
           set +a
@@ -86,14 +86,14 @@ services:
           memory: "${SPARKD__MEMORY:-4gb}"
 
   sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
+    image: bluetradingsystems/linux_dotnet8
     volumes:
       - type: volume
         read_only: true
         source: sparkd_data
         target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
         volume:
-          subpath: logging
+          subpath: logging.sparkd
     command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
 
   sparkd-ping-test:
@@ -104,14 +104,14 @@ services:
     build:
       context: .
       dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
+        FROM bluetradingsystems/linux_dotnet8
         USER "root"
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
 
   pubsub:
     volumes:
-      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/PubSubServer/data
+      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
     build:
       dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
       args:
@@ -121,20 +121,20 @@ services:
       - 22019:22019
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/PubSubServer
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" ]; then
-          mv "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
+        if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
+          mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env'
+        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
           set -a
-          . /home/bts/.local/share/bts/PubSubServer/conf/.env
+          . /home/bts/.local/share/bts/pubsub/conf/.env
           set +a
         fi
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env.local'
+        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
           set -a
-          . /home/bts/.local/share/bts/PubSubServer/conf/.env.local
+          . /home/bts/.local/share/bts/pubsub/conf/.env.local
           set +a
         fi
         ./PubSubServer"
@@ -144,8 +144,8 @@ services:
 
   md-publisher:
     volumes:
-      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/DataStreamPublisher/data
-      - historic_md:/home/bts/.local/share/bts/databento
+      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
+      - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
     build:
       dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
       args:
@@ -153,20 +153,20 @@ services:
         - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/DataStreamPublisher
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" ]; then
-          mv "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
+        if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
+          mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env'
+        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
           set -a
-          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env
+          . /home/bts/.local/share/bts/md_publisher/conf/.env
           set +a
         fi
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local'
+        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
           set -a
-          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local
+          . /home/bts/.local/share/bts/md_publisher/conf/.env.local
           set +a
         fi
       dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet8
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile

@@ -55,14 +55,16 @@ RUN \
    clangd-$clang_major
 
 # CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
+RUN set -eux \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends ca-certificates curl gpg \
+  && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
+    | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
+  && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    > /etc/apt/sources.list.d/kitware.list \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends cmake \
+  && rm -rf /var/lib/apt/lists/*
 
 # Dotnet
 RUN apt-get update && apt-get install -y dotnet8
@@ -80,9 +82,14 @@ RUN mkdir -p /etc/apt/keyrings \
 # Emumba (deps)
 RUN apt-get update && apt-get -y install rapidjson-dev
 # Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
+ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
+RUN --mount=type=secret,id=bitbucket_access_token \
+  set -eux \
+    && apt-get update \
+    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
+    && curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
+    && apt-get install -y /tmp/emumba-deribit-connector.deb \
+    && rm -f /tmp/emumba-deribit-connector.deb
 
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -90,6 +97,18 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
+# Rakurai
+ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
+RUN --mount=type=secret,id=bitbucket_access_token \
+  set -eux \
+    && apt-get update \
+    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
+    && curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
+    # Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
+    # spdlog headers.
+    && apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
+    && rm -f /tmp/rakuirai.deb
+
 # Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \

global.json

@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "6.0.200",
+    "rollForward": "latestMajor"
+  }
+}

requirements.txt

@@ -18,10 +18,10 @@ MarkupSafe==3.0.2
 mccabe==0.6.1
 numpy==2.2.6
 platformdirs==2.3.0
-protobuf==4.21.12
+protobuf==6.33.4
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
+pybind11==3.0.1
+pybind11-stubgen==2.5.5
 pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20

spark_client.sln

@@ -1,164 +1,414 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
-		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
-	EndGlobalSection
-EndGlobal
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{00D21F82-61F4-467F-B125-268018A185E5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{513A3F0D-46B5-4AC2-B700-8671933F6347}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{D419D996-9870-4FF5-82AD-08370D1C945F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.ActiveCfg = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.Build.0 = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.ActiveCfg = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.Build.0 = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.ActiveCfg = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.Build.0 = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.Build.0 = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.ActiveCfg = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.Build.0 = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.ActiveCfg = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.Build.0 = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.ActiveCfg = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.Build.0 = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.ActiveCfg = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.Build.0 = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.Build.0 = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.ActiveCfg = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.Build.0 = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{00D21F82-61F4-467F-B125-268018A185E5} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{513A3F0D-46B5-4AC2-B700-8671933F6347} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{D419D996-9870-4FF5-82AD-08370D1C945F} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+	EndGlobalSection
+EndGlobal

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -11,10 +11,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ArithmeticExpression">
     <Scanner Include="ArithmeticExpressionTokens.lex" />

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -5,9 +5,5 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
+  
 </Project>

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,23 +6,80 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var cmp1 = value1.Cast<IComparable>();
-      var cmp2 = value2.Cast<IComparable>();
+      var obj1 = value1.GetValue(env);
+      var obj2 = value2.GetValue(env);
+      if (obj1 is null) {
+         if (obj2 is null) {
+            return 0;
+         }
+         return -1;
+      }
+      if (obj2 is null) {
+         return 1;
+      }
+      return compareObjs(obj1, obj2);
+   }
+
+   static int compareObjs(object obj1, object obj2) {
       try {
-         var v1 = cmp1.GetValue(env);
-         var v2 = cmp2.GetValue(env);
-         return v1.GetType().IsNumeric() ?
-            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
+         var typ1 = obj1.GetType();
+         var typ2 = obj2.GetType();
+         bool areSameTypes = typ1 == typ2;
+         if (areSameTypes) {
+            return ((IComparable)obj1).CompareTo(obj2);
+         }
+         if (typ1.IsNumeric()) {
+            return tryCompareNumeric(obj1, obj2);
+         } else if (typ2.IsNumeric()) {
+            return -tryCompareNumeric(obj2, obj1);
+         } else if (typ1.IsEnum) {
+            if (typ2 == typeof(string)) {
+               return -((string)obj2).CompareTo(obj1.ToString());
+            }
+         } else if (typ1 == typeof(string)) {
+            return ((string)obj1).CompareTo(obj2.ToString());
+         } else if (typ1 == typeof(DateTime)) {
+            if (typ2 == typeof(TimeSpan)) {
+               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
+            }
+         } else if (typ1 == typeof(TimeSpan)) {
+            if (typ2 == typeof(DateTime)) {
+               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
+            }
+         }
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
+      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
+   }
+
+   static int tryCompareNumeric(object numericObj1, object obj2) {
+      var typ2 = obj2.GetType();
+      if (typ2.IsNumeric()) {
+         return new NumericComparer().Compare(numericObj1, obj2);
+      } else if (typ2.IsEnum) {
+         var undTyp2 = typ2.GetEnumUnderlyingType();
+         if (undTyp2 == typeof(byte)) {
+            return new NumericComparer().Compare(numericObj1, (byte)obj2);
+         }
+         if (undTyp2 == typeof(short)) {
+            return new NumericComparer().Compare(numericObj1, (short)obj2);
+         }
+         if (undTyp2 == typeof(int)) {
+            return new NumericComparer().Compare(numericObj1, (int)obj2);
+         }
+         if (undTyp2 == typeof(long)) {
+            return new NumericComparer().Compare(numericObj1, (long)obj2);
+         }
+      }
+      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -11,10 +11,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
   
   <ItemGroup>
     <Scanner Include="BooleanExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -10,11 +10,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnExpression">
     <Scanner Include="ColumnExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -10,11 +10,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnSortExpression">
     <Scanner Include="ColumnSortExpressionTokens.lex" />

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -10,9 +10,18 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnMapExpression">
     <Scanner Include="ColumnMapExpressionTokens.lex" />

src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs

@@ -1,118 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts.collections;
-using bts.utils;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-public class BlueAddIndexColumnTable : IBlueRowCollection {
-   readonly IBlueTableSchema schema;
-   readonly IList<object?>[] columnValues;
-   readonly int count;
-
-   BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
-      this.schema = schema;
-      this.columnValues = columnValues;
-      this.count = count ?? columnValues[0].Count;
-   }
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public int Count => count;
-
-   public IBlueRow this[int id] =>
-      id > 0 && id <= count
-         ? GetRow(id - 1)
-         : throw new KeyNotFoundException();
-
-   public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
-      if (columnInfos.Any(c => c.colName == "id")) {
-         throw new ArgumentException("'id' column already exists", nameof(columnInfos));
-      }
-      if (columnInfos.Length != columnValues.Length) {
-         throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
-      }
-      int count = columnValues.FirstOrDefault()?.Count ?? 0;
-      var origCols = columnInfos;
-      var origColValues = columnValues;
-      columnInfos = new ColumnInfo[origCols.Length + 1];
-      columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
-      columnValues = new IList<object?>[origCols.Length + 1];
-      columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
-      for (int i = 1; i < columnInfos.Length; i++) {
-         columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
-         columnValues[i] = origColValues[i - 1];
-      }
-      var schema = new BlueTableSchema(columnInfos, tableName);
-      return new(schema, columnValues, count);
-   }
-
-   public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
-      int count = columnValues.FirstOrDefault()?.Count ?? 0;
-      int? idColIdx = schema.TryResolveIndices("id");
-      var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
-      var columnInfos = schema.columnInfo;
-      if (idColIdx.HasValue) {
-         // Replace existing id column. columnValues may or may not contain the old ids (if its
-         // length is one less column than that of columnInfos, it doesn't contain them).
-         switch (columnValues.Length - schema.columnInfo.Length) {
-            case 0:
-               columnValues[idColIdx.Value] = idColValues;
-               break;
-            case -1:
-               columnValues = ArrayHelper.InsertExact(columnValues, idColIdx.Value, true, idColValues);
-               break;
-            default:
-               throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
-         }
-      } else {
-         if (columnValues.Length != columnInfos.Length) {
-            throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
-         }
-         var origColInfos = columnInfos;
-         columnInfos = new ColumnInfo[columnInfos.Length + 1];
-         columnInfos[0] = new("id", 0, typeof(int));
-         for (int i = 1; i < columnInfos.Length; i++) {
-            var c = origColInfos[i - 1];
-            columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
-         }
-         columnValues = ArrayHelper.InsertExact(columnValues, 0, true, idColValues);
-      }
-      return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
-   }
-
-   public bool containsID(int id) =>
-      id > 0 && id <= count;
-
-   public IList<object?> GetColumn(int index) =>
-      columnValues[index];
-
-   public IBlueRow GetRow(int index) =>
-      new BlueRowColumnWrapper(schema, index, columnValues);
-
-   public int IndexOf(IBlueRow item) {
-      int id = item.Id();
-      return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
-         ? id - 1
-         : -1;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      int count = Count;
-      for (int i = 0; i < count; i++) {
-         yield return GetRow(i);
-      }
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}

src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs

@@ -1,56 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using mbts.utils;
-
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-public class BlueIndexedColumnTable : IBlueRowCollection {
-   readonly IBlueTableSchema schema;
-   readonly IList<object?>[] columnValues;
-   readonly Dictionary<int, int> rowIdToIndex;
-
-   public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
-      this.schema = schema;
-      this.columnValues = columnValues;
-      int idColumnIndex = schema.ResolveIndices("id");
-      rowIdToIndex = columnValues[idColumnIndex]
-         .Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
-         .ToDictionary();
-   }
-
-   public IBlueRow this[int id] =>
-      rowIdToIndex.TryGetValue(id, out var idx)
-         ? GetRow(idx)
-         : throw new KeyNotFoundException();
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public int Count => rowIdToIndex.Count;
-
-   public bool containsID(int id) =>
-      rowIdToIndex.ContainsKey(id);
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      int count = Count;
-      for (int i = 0; i < count; i++) {
-         yield return GetRow(i);
-      }
-   }
-
-   public IBlueRow GetRow(int index) =>
-      new BlueRowColumnWrapper(schema, index, columnValues);
-
-   public int IndexOf(IBlueRow item) =>
-      rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
-         ? idx
-         : -1;
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}

src/DataStreamSuite/DataStream/BlueOrb/BlueRowColumnWrapper.cs

@@ -1,21 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using System.Collections.Generic;
-
-public class BlueRowColumnWrapper : IBlueRow {
-   readonly int index;
-   readonly IList<object?>[] columns;
-
-   public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
-      this.schema = schema;
-      this.index = index;
-      this.columns = columns;
-   }
-
-   public object? this[int index] =>
-      columns[index][this.index];
-
-   public IBlueTableSchema schema { get; }
-}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,9 +1,7 @@
-using System;
-
 namespace BCG {
    [DataStreamUserRespresentation("CountStream")]
    public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
+      IDataPacket? myLastFiredDataPacket;
       private IBlueTableSchema mySchema = new BlueTableSchema("count",
          new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("count", 1, typeof(int)));
@@ -14,6 +12,9 @@ namespace BCG {
          myVS = vs;
       }
 
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
+
       protected override void myOnFirstSubscriber() {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
@@ -21,7 +22,6 @@ namespace BCG {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
-
       private void myHandleChanges(object newValue) {
          myFireEvent(
             myLastFiredDataPacket =
@@ -36,11 +36,8 @@ namespace BCG {
 
       private IBlueRowCollection myGetRows(object dp, int id) {
          BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+         retVal.addRow(new BlueRow(mySchema, id, ((IDataPacket)dp).newRows.Count));
          return retVal;
       }
-
-
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -15,6 +15,7 @@ namespace bts.BlueOrb {
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_year");
             return rv;
          };
          ViewStream rs = new MapStream(parent_stream,
@@ -30,6 +31,7 @@ namespace bts.BlueOrb {
             rv.setValue($"{inp_col}_hour", dt.Hour);
             rv.setValue($"{inp_col}_minute", dt.Minute);
             rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            rv.setValue($"{inp_col}_year", dt.Date.Year);
             return rv;
          });
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,5 +1,8 @@
 namespace bts.BlueOrb {
    using BCG;
+
+   using Bts.DataStreamCore;
+
    using System;
    using System.Text.RegularExpressions;
 
@@ -21,22 +24,24 @@ namespace bts.BlueOrb {
 
          #region IStreamFilter Members
          public bool filter(IBlueRow dr) {
-            bool rv = true;
             for (int i = 0; i < myCols.Length; i++) {
-               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-               if (!rv)
-                  return rv;
+               var val = dr[myCols[i]];
+               var str = val is DBNull
+                  ? string.Empty
+                  : (val?.ToString() ?? string.Empty);
+               if (!myRegExps[i].IsMatch(str)) {
+                  return false;
+               }
             }
-            return rv;
+            return true;
          }
          #endregion
       }
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
+      BlueTable? myLastFiredRows;
+      IBlueRowCollection? myLastReceivedRows;
       private IStreamFilter myFilter;
       private ViewStream myParentStream;
       private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -52,12 +57,11 @@ namespace bts.BlueOrb {
 
 
       public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         myParentStream = vs;
+         myFilter = filter;
+         myFilterString = filter.ToString() ?? string.Empty;
+         if (filter is IDynamicStreamFilter dynFilter) {
+            dynFilter.onFilterChange += myHandleFilterChange;
          }
       }
 
@@ -86,7 +90,7 @@ namespace bts.BlueOrb {
       }
 
       protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
+         var dp = (IDataPacket)o;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -94,23 +98,27 @@ namespace bts.BlueOrb {
 
       private void myHandleCallbacks(object newValue) {
          DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
+         IDataPacket arrivedPacket = (IDataPacket)newValue;
          myLastReceivedRows = arrivedPacket.newRows;
          if (myLastFiredRows != null) {
             retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
                new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
+            var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
+            foreach (IBlueRow r in arrivedPacket.deletedRows) {
+               if (myLastFiredRows.containsID(r.Id())) {
                   myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
+                  deletedRows.Value.addRow(r);
                }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
+            }
+            var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
+            foreach (IBlueRow r in arrivedPacket.addedRows) {
                if (myFilter.filter(r)) {
                   myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
+                  addedRows.Value.addRow(r);
                }
+            }
          } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            myLastFiredRows = new BlueTable(arrivedPacket.Schema);
             retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
             foreach (IBlueRow r in arrivedPacket.newRows)
                if (myFilter.filter(r))

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -1,12 +1,13 @@
-using System;
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
 
 namespace bts.BlueOrb {
    [DataStreamUserRespresentation("GroupByStream")]
@@ -14,20 +15,21 @@ namespace bts.BlueOrb {
 
       public interface INonIncGroupByEvaluator {
          // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
+         IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[]? get_group_by_cols();
       }
 
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
+         Func<string[]?> _groupby_cols_func;
          Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
                                           Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
+
+         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable? rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
                List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
@@ -39,17 +41,19 @@ namespace bts.BlueOrb {
             }
             return rv;
          }
-         public string[] get_group_by_cols() {
+
+         public string[]? get_group_by_cols() {
             return _groupby_cols_func();
          }
 
       }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
+      ViewStream? _main_stream;
+      INonIncGroupByEvaluator? _groupby_func;
+      BlueRowHasher? _row_hasher;
+      string[]? _groupby_cols;
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
+      IDisposable? subscription;
 
       public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -58,7 +62,7 @@ namespace bts.BlueOrb {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+      public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
@@ -68,34 +72,42 @@ namespace bts.BlueOrb {
       // this is the main constructor that does everything
       public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            if (String.IsNullOrEmpty(groupby_cols))
-               return null;
-            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
-         };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
             // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
             if (rv == null)
                return new List<IBlueRow>();
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
          mySetRealSource(rs);
+
+         string[]? groupby_cols_func() {
+            if (string.IsNullOrEmpty(groupby_cols)) {
+               return null;
+            }
+            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
+         }
       }
       public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
       this(parent_str, String.Empty, standard_group_by_text) {
       }
 
+      protected override void myOnFirstSubscriber() {
+         Log.isnull(subscription);
+         subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
+      }
 
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+      protected override void myOnLastSubscriber() {
+         subscription?.Dispose();
+         subscription = null;
+      }
 
       private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
+         Log.notnull(_groupby_func);
+         var dp = (IDataPacket)nv;
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -109,7 +121,7 @@ namespace bts.BlueOrb {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
          if (total_state == null)
             // mayeb returna n empty table ?
             return;
@@ -121,7 +133,10 @@ namespace bts.BlueOrb {
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         var deletedRows = myLastValue is null
+            ? new BlueTable(total_state.schema)
+            : ((IDataPacket)myLastValue).newRows;
+         myFireEvent(new DataPacket(total_state, total_state, deletedRows));
       }
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,41 +1,44 @@
 namespace bts.BlueOrb {
    using BCG;
-   using RowHash = System.Int32;
+
+   using Bts.DataStream;
+
+   using Spark.Core.Diagnostics;
+
    using System;
    using System.Collections.Generic;
+   using System.Diagnostics.CodeAnalysis;
    using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   using System.Collections;
-   using System.Data.Entity.Core.Mapping;
+
+   using RowHash = System.Int32;
 
    [DataStreamUserRespresentation("JoinStream")]
    public class SimpleJoinStream : ViewStream {
       public class JoinBookkeepingData {
          // initialized in cons
-         private readonly List<string> _join_col_names;
-         private readonly bool _is_left;
+         readonly string[] _join_col_names;
          private readonly BlueRowHasher _hasher;
          // initialized on first udpate         
-         private IBlueTableSchema _schema;
-         private int[] _join_cols;
+         IBlueTableSchema? _schema;
+         int[]? _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
 
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
-            _is_left = is_left;
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
+            _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
             _hasher = row_hasher;
          }
 
-         public IBlueTableSchema schema => _schema;
+         [NotNullIfNotNull(nameof(_schema))]
+         public IBlueTableSchema? schema => _schema;
 
          public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
 
-         public List<IBlueRow> get_matching_rows(int hash) {
-            List<IBlueRow> rv = null;
-            _hash_row_map.TryGetValue(hash, out rv);
-            return rv;
-         }
+         [MemberNotNull(nameof(_schema))]
+         public IBlueTableSchema RequireSchema() =>
+            ExceptionAssert.MemberNotNull(_schema);
+
+         public List<IBlueRow>? get_matching_rows(int hash) =>
+            _hash_row_map.GetValueOrDefault(hash);
 
          private RowHash _get_hash(IBlueRow r) {
             if (_join_cols == null || _join_cols.Length == 0) {
@@ -44,16 +47,15 @@
             return _hasher.get_hash(r, _join_cols);
          }
 
-         public void update_data(DataPacket dp) {
+         public void update_data(IDataPacket dp) {
             if (!ready) {
                _schema = dp.newRows.schema;
-               _join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+               _join_cols = _schema.ResolveIndices(_join_col_names);
             }
             _hash_row_map.Clear();
             foreach (IBlueRow r in dp.newRows) {
                int hash = _get_hash(r);
-               List<IBlueRow> hash_rows = null;
-               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
+               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
                   hash_rows = new List<IBlueRow>();
                   _hash_row_map.Add(hash, hash_rows);
                }
@@ -61,6 +63,7 @@
             }
          }
 
+         [MemberNotNullWhen(true, nameof(_schema))]
          public bool ready {
             get {
                return _schema != null;
@@ -71,8 +74,7 @@
 
       private ViewStream _ls, _rs;
       private JoinBookkeepingData _ldata, _rdata;
-      private ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema = null;
+      IBlueTableSchema? _schema;
 
       public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
          this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
@@ -85,8 +87,8 @@
          Log.assert(lj_col_list.Count == rj_col_list.Count);
          int n_cols = lj_col_list.Count;
          BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
+         _ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
+         _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
       }
 
       protected override void myOnFirstSubscriber() {
@@ -95,13 +97,13 @@
       }
 
       private void _ls_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = (IDataPacket)dp_obj;
          _ldata.update_data(dp);
          _form_and_fire();
       }
 
       private void _rs_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = (IDataPacket)dp_obj;
          _rdata.update_data(dp);
          _form_and_fire();
       }
@@ -111,20 +113,24 @@
          _rs.onChange -= _rs_onchange;
       }
 
+      [MemberNotNull(nameof(_schema))]
       private void _build_schema() {
          if (_schema != null)
             return;
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
-         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
-            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
+         var lschema = Log.notnull(_ldata.schema);
+         var rschema = Log.notnull(_rdata.schema);
+         var cis = new List<ColumnInfo> {
+            new("id", 0, TypeCode.Int32)
+         };
+         foreach (ColumnInfo ci in lschema.columnInfo) {
+            string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
-            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
+         foreach (ColumnInfo ci in rschema.columnInfo) {
+            string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
+         _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
       }
 
       private void _form_and_fire() {
@@ -133,34 +139,11 @@
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
 
-         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.schema.columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         };
-         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
-            object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         };
          // this is where the join happens
          foreach (var kv in _ldata.rows) {
             RowHash hash = kv.Key;
             List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
+            List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
 
             foreach (IBlueRow lr in l_rows) {
                if (r_rows != null) {
@@ -179,8 +162,33 @@
          foreach (object[] rval in res_rows) {
             rv.addRow(new BlueRow(_schema, rval));
          }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
          myFireEvent(dp);
+
+         object[] build_row_array(int id, IBlueRow l_row) {
+            object[] rv = new object[_schema.columnNames.Length];
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
+               rv[i] = DBNull.Value;
+            }
+            rv[0] = id;
+            copy_row(rv, l_row, true);
+            return rv;
+         }
+
+         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
+            int offset = 1;
+            JoinBookkeepingData sdata = _ldata;
+            if (is_left) {
+               offset = 1;
+               sdata = _ldata;
+            } else {
+               sdata = _rdata;
+               offset = 1 + _ldata.RequireSchema().columnInfo.Length;
+            }
+            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
+               rval[offset + i] = r[i];
+            }
+         }
       }
    }
 

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -43,7 +43,7 @@ namespace bts.BlueOrb {
             _last_fire_time = DateTime.Now;
             IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
             _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table = _table.Add(nr);
+            _table.add_row(nr);
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
             myFireEvent(dp);
          }

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -45,8 +45,8 @@ namespace bts.BlueOrb {
       }
 
       // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
+      public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count == 0) {
             return null;
          }
 
@@ -73,8 +73,8 @@ namespace bts.BlueOrb {
       }
 
       // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
+      public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
+         if (rows.Count == 0) {
             return null;
          }
          IBlueRow ir = rows.ElementAt(start_idx);
@@ -104,8 +104,8 @@ namespace bts.BlueOrb {
       // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
       // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
 
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
+      public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count == 0) {
             return null;
          }
          IBlueRow ir = rolling_rows.ElementAt(0);
@@ -156,7 +156,6 @@ namespace bts.BlueOrb {
                   double first = col_vals.First();
                   double last = col_vals.Last();
                   return ((first - last) / last) * 100.0;
-                  return col_vals.First() - col_vals.Last();
                }
             default:
                // TODO

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -1,8 +1,7 @@
 namespace bts.BlueOrb {
    using BCG;
 
-   using System.Collections.Generic;
-   using System.Net;
+   using System;
 
    [DataStreamUserRespresentation("MultiRowRollingMathStream")]
    public class MultiRowRollingMathStream : ViewStream {
@@ -24,11 +23,12 @@
       }
 
       private void _parent_str_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = (IDataPacket)dp_obj;
          var rows = dp.newRows.to_list();
-         BlueTable out_table = null;
+         BlueTable? out_table = null;
          for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
+            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
+               ?? throw new InvalidOperationException("group_by_compute returned null row");
             if (out_table == null)
                out_table = new BlueTable(out_r.schema);
             out_table.addRow(out_r);

src/DataStreamSuite/DataStream/BlueOrb/Parquet/IParquetColumnReader.cs

@@ -1,29 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts.collections;
-
-using System;
-using System.Collections.Generic;
-
-public interface IParquetColumnReader : IDisposable {
-   IList<object?> Read();
-   IList<object?> Read(int? start, int? stop);
-}
-
-public interface IParquetColumnReader<TLogical, TPhysical> : IParquetColumnReader {
-   new IList<TLogical?> Read();
-   new IList<TLogical?> Read(int? start, int? stop);
-
-   IList<object?> IParquetColumnReader.Read() =>
-      Read().WrapSelector(v => (object?)v, null);
-
-   IList<object?> IParquetColumnReader.Read(int? start, int? stop) =>
-      Read(start, stop).WrapSelector(v => (object?)v, null);
-}
-
-public interface IParquetColumnReader<T> : IParquetColumnReader<T, T> {
-   new IList<T?> Read();
-
-   IList<object?> IParquetColumnReader.Read() =>
-      Read().WrapSelector(v => (object?)v, null);
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetAdapter.cs

@@ -1,507 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts;
-using bts.collections;
-
-using mbts.utils;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-
-public static class ParquetAdapter {
-   public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) {
-      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
-      var colReader = rowGroupReader.Column(index);
-      return getColumnReader(colReader, rowCount, colType: colType);
-   }
-
-   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader) {
-      var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
-      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
-      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
-      for (int i = 0; i < colValues.Length; i++) {
-         colValues[i] = new List<object?>(rowCount);
-      }
-      int rowGroupCount = fileReader.FileMetaData.NumRowGroups;
-      for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) {
-         using var rowGroupReader = fileReader.RowGroup(rowGroup);
-         rowGroupReader.ReadColumnsTo(ref colValues, colTypes);
-      }
-      return colValues;
-   }
-
-   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) {
-      if (!start.HasValue && !stop.HasValue) {
-         return ReadColumns(fileReader);
-      }
-      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
-      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
-      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
-      start ??= 0;
-      stop ??= rowCount;
-      var sliceLength = stop.Value - start.Value;
-      for (int i = 0; i < colValues.Length; i++) {
-         colValues[i] = new List<object?>(sliceLength);
-      }
-      int numRowGroups = fileReader.FileMetaData.NumRowGroups;
-      int? numRowsPerGroup = null;
-      int rowIdx = 0;
-      for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) {
-         if (stop.Value <= rowIdx) {
-            break;
-         }
-         using var rowGroupReader = fileReader.RowGroup(rowGroup);
-         int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows);
-         numRowsPerGroup ??= numRowsInGroup;
-         if (rowGroup < numRowGroups - 1) {
-            Log.assert(numRowsInGroup == numRowsPerGroup.Value);
-         } else {
-            Log.assert(numRowsInGroup <= numRowsPerGroup.Value);
-         }
-         int startIdx = start.Value - rowIdx;
-         int stopIdx = stop.Value - rowIdx;
-         rowIdx += numRowsInGroup;
-         if (startIdx >= numRowsInGroup) {
-            continue;
-         }
-         startIdx = Math.Max(0, startIdx);
-         stopIdx = Math.Min(stopIdx, numRowsInGroup);
-         int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx);
-         Log.assert(n == (stopIdx - startIdx));
-      }
-      return colValues;
-   }
-
-   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes) {
-      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
-      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
-         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
-         columnValues[colIdx].AddRange(colReader.Read());
-      }
-      return rowCount;
-   }
-
-   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes, int? start, int? stop) {
-      if (!start.HasValue && !stop.HasValue) {
-         return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes);
-      }
-      int numRows = checked((int)rowGroupReader.MetaData.NumRows);
-      if (start.HasValue && start.Value >= numRows) {
-         return 0;
-      }
-      int? readRowCount = null;
-      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
-         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
-         var colVals = colReader.Read(start, stop);
-         readRowCount ??= colVals.Count;
-         Log.assert(readRowCount == colVals.Count);
-         columnValues[colIdx].AddRange(colVals);
-      }
-      return readRowCount ?? throw new ArgumentException("Row group has no columns", nameof(rowGroupReader));
-   }
-
-   static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) {
-      var desc = reader.ColumnDescriptor;
-      var logicalType = desc.LogicalType;
-      return logicalType.Type switch {
-         LogicalTypeEnum.Int => IntReader(),
-         LogicalTypeEnum.String => new ParquetColumnReader<string?>(reader, bufferSize),
-         LogicalTypeEnum.Date => new ParquetColumnReader<DateTime?>(reader, bufferSize),
-         LogicalTypeEnum.Decimal => DecimalReader(),
-         LogicalTypeEnum.Timestamp => TimestampReader(),
-         LogicalTypeEnum.None => DefaultReader(),
-         _ => throw new NotImplementedException(),
-      };
-
-      IParquetColumnReader DefaultReader() {
-         return desc.PhysicalType switch {
-            PhysicalType.Boolean => new ParquetColumnReader<bool?>(reader, bufferSize),
-            PhysicalType.Int32 => new ParquetColumnReader<int?>(reader, bufferSize),
-            PhysicalType.Int64 => new ParquetColumnReader<long?>(reader, bufferSize),
-            PhysicalType.Float => new ParquetColumnReader<float?>(reader, bufferSize),
-            PhysicalType.Double => new ParquetColumnReader<double?>(reader, bufferSize),
-            _ => throw new NotImplementedException(),
-         };
-      }
-
-      IParquetColumnReader TimestampReader() {
-         var tlt = ExceptionAssert.of_type<TimestampLogicalType>(logicalType);
-         return new ParquetColumnReader<DateTimeNanos?, DateTime?>(reader, dtn => dtn?.DateTime, bufferSize);
-      }
-
-      IParquetColumnReader IntReader() {
-         var ilt = ExceptionAssert.of_type<IntLogicalType>(logicalType);
-         if (!ilt.IsSigned) {
-            if (colType is null) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(byte)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, byte?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(ushort)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, ushort?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(uint)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, uint?>(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(ulong)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            }
-            throw new NotImplementedException($"Expected signed int column type, but got {colType}");
-         } else {
-            if (colType is null) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(sbyte)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<short?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  32 => new ParquetColumnReader<int?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<long?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(short)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<int?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<long?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(int)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<short?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<long?, int?>(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(long)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<short?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<int?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            }
-            throw new NotImplementedException($"Expected unsigned int column type, but got {colType}");
-         }
-      }
-
-      IParquetColumnReader DecimalReader() {
-         var dlt = ExceptionAssert.of_type<DecimalLogicalType>(logicalType);
-         return new ParquetColumnReader<double?>(reader, bufferSize);
-      }
-   }
-
-   public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) {
-      return typeCode switch {
-         TypeCode.Boolean => new ColumnWriter<bool?>(writer, bufferSize),
-         TypeCode.SByte => new ColumnWriter<sbyte?>(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int16 => new ColumnWriter<short?>(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int32 => new ColumnWriter<int?>(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int64 => new ColumnWriter<long?>(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Byte => new ColumnWriter<byte?>(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt16 => new ColumnWriter<ushort?>(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt32 => new ColumnWriter<uint?>(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt64 => new ColumnWriter<ulong?>(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Single => new ColumnWriter<float?>(writer, bufferSize),
-         TypeCode.Double => new ColumnWriter<double?>(writer, bufferSize),
-         TypeCode.DateTime => new ColumnWriter<DateTimeNanos?>(writer, bufferSize, ConvertDateTimeObj),
-         TypeCode.String => new ColumnWriter<string?>(writer, bufferSize),
-         _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
-      };
-
-      static DateTimeNanos? ConvertDateTimeObj(object? obj) {
-         if (obj is DateTime dt) {
-            return new(dt);
-         }
-         if (obj is DateTimeNanos dtn) {
-            return dtn;
-         }
-         throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos");
-      }
-   }
-
-   public static Func<object?, sbyte?> GetInt8Converter(IntLogicalType logicalType) =>
-      GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, sbyte?> GetInt8Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj as sbyte?,
-            16 => obj => obj is short v ? checked((sbyte)v) : null,
-            32 => obj => obj is int v ? checked((sbyte)v) : null,
-            64 => obj => obj is long v ? checked((sbyte)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? checked((sbyte)v) : null,
-            16 => obj => obj is ushort v ? checked((sbyte)v) : null,
-            32 => obj => obj is uint v ? checked((sbyte)v) : null,
-            64 => obj => obj is ulong v ? checked((sbyte)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, short?> GetInt16Converter(IntLogicalType logicalType) =>
-      GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, short?> GetInt16Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj as short?,
-            32 => obj => obj is int v ? checked((short)v) : null,
-            64 => obj => obj is long v ? checked((short)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? checked((short)v) : null,
-            32 => obj => obj is uint v ? checked((short)v) : null,
-            64 => obj => obj is ulong v ? checked((short)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, int?> GetInt32Converter(IntLogicalType logicalType) =>
-      GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, int?> GetInt32Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj is short v ? v : null,
-            32 => obj => obj as int?,
-            64 => obj => obj is long v ? checked((int)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? checked((int)v) : null,
-            64 => obj => obj is ulong v ? checked((int)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, long?> GetInt64Converter(IntLogicalType logicalType) =>
-      GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, long?> GetInt64Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj is short v ? v : null,
-            32 => obj => obj is int v ? v : null,
-            64 => obj => obj as long?,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? checked((long)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, byte?> GetUInt8Converter(IntLogicalType logicalType) =>
-      GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, byte?> GetUInt8Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((byte)v) : null,
-            16 => obj => obj is short v ? checked((byte)v) : null,
-            32 => obj => obj is int v ? checked((byte)v) : null,
-            64 => obj => obj is long v ? checked((byte)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj as byte?,
-            16 => obj => obj is ushort v ? checked((byte)v) : null,
-            32 => obj => obj is uint v ? checked((byte)v) : null,
-            64 => obj => obj is ulong v ? checked((byte)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, ushort?> GetUInt16Converter(IntLogicalType logicalType) =>
-      GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, ushort?> GetUInt16Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((ushort)v) : null,
-            16 => obj => obj is short v ? checked((ushort)v) : null,
-            32 => obj => obj is int v ? checked((ushort)v) : null,
-            64 => obj => obj is long v ? checked((ushort)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? checked((ushort)v) : null,
-            64 => obj => obj is ulong v ? checked((ushort)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, uint?> GetUInt32Converter(IntLogicalType logicalType) =>
-      GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, uint?> GetUInt32Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((uint)v) : null,
-            16 => obj => obj is short v ? checked((uint)v) : null,
-            32 => obj => obj is int v ? checked((uint)v) : null,
-            64 => obj => obj is long v ? checked((uint)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? checked((uint)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, ulong?> GetUInt64Converter(IntLogicalType logicalType) =>
-      GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, ulong?> GetUInt64Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((ulong)v) : null,
-            16 => obj => obj is short v ? checked((ulong)v) : null,
-            32 => obj => obj is int v ? checked((ulong)v) : null,
-            64 => obj => obj is long v ? checked((ulong)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? v : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static string ColumnEnumTypeKey(string colName) =>
-      $"{colName}.EnumType";
-
-   public static KeyValuePair<string, string> ColumnEnumTypeEntry(string colName, Type enumType) {
-      if (enumType.IsNullable()) {
-         enumType = enumType.GetGenericArguments()[0];
-      }
-      if (!enumType.IsEnum) {
-         throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType));
-      }
-      return new(ColumnEnumTypeKey(colName), enumType.ToString());
-   }
-
-   public static Type? GetColumnEnumType(IReadOnlyDictionary<string, string>? keyValueMetadata, string columnName) {
-      if (keyValueMetadata is not null) {
-         var key = ColumnEnumTypeKey(columnName);
-         if (keyValueMetadata.TryGetValue(key, out var typeName)) {
-            return Type.GetType(typeName);
-         }
-      }
-      return null;
-   }
-
-   public interface IColumnWriter : IDisposable {
-      void Write(IReadOnlyCollection<object?> values);
-   }
-
-   public interface IColumnWriter<T> : IColumnWriter {
-      void Write(IReadOnlyCollection<T?> values);
-   }
-
-   class ColumnWriter<T> : IColumnWriter<T> {
-      readonly T?[] buffer;
-      readonly LogicalColumnWriter<T> writer;
-      readonly Func<object?, T?> convert;
-
-      public ColumnWriter(ColumnWriter writer, int bufferSize)
-         : this(writer, new T?[bufferSize]) { }
-
-      public ColumnWriter(ColumnWriter writer, int bufferSize, Func<object?, T?> convert)
-         : this(writer, new T?[bufferSize], convert) { }
-
-      public ColumnWriter(ColumnWriter writer, T?[] buffer)
-         : this(writer, buffer, obj => (T?)obj) { }
-
-      public ColumnWriter(ColumnWriter writer, T?[] buffer, Func<object?, T?> convert) {
-         this.buffer = buffer;
-         this.writer = writer.LogicalWriter<T>();
-         this.convert = convert;
-      }
-
-      public void Write(IReadOnlyCollection<T?> values) {
-         if (buffer.Length == 0) {
-            throw new InvalidOperationException("Cannot write because buffer length is 0");
-         }
-         var count = values.Count;
-         var nPages = count / buffer.Length;
-         using var en = values.GetEnumerator();
-         for (int pg = 0; pg < nPages; pg++) {
-            for (int i = 0; i < buffer.Length; i++) {
-               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
-               buffer[i] = en.Current;
-            }
-            writer.WriteBatch(buffer!);
-         }
-         var remainder = count - nPages * buffer.Length;
-         if (remainder != 0) {
-            for (int i = 0; i < remainder; i++) {
-               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
-               buffer[i] = en.Current;
-            }
-            writer.WriteBatch(buffer!, 0, remainder);
-         }
-      }
-
-      public void Dispose() =>
-         writer.Dispose();
-
-      void IColumnWriter.Write(IReadOnlyCollection<object?> values) =>
-         Write(ReadOnlySelectorCollection.Create(values, convert));
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetColumnReader.cs

@@ -1,111 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts;
-using bts.collections;
-
-using ParquetSharp;
-
-using System;
-using System.Collections.Generic;
-
-class ParquetColumnReader<T> : IParquetColumnReader<T> {
-   protected readonly LogicalColumnReader<T> reader;
-   protected readonly T?[] buffer;
-
-   public ParquetColumnReader(ColumnReader reader, int bufferSize)
-      : this(reader, new T?[bufferSize]) { }
-
-   protected ParquetColumnReader(ColumnReader reader, T?[] buffer) {
-      this.buffer = buffer;
-      this.reader = reader.LogicalReader<T>();
-   }
-
-   ~ParquetColumnReader() =>
-      Dispose();
-
-   public IList<T?> Read() {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      int n = reader.ReadBatch(buffer!);
-      return buffer.WrapSubRange(0, n);
-   }
-
-   public IList<T?> Read(int? start, int? stop) {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      start ??= 0;
-      stop ??= buffer.Length;
-      int length = stop.Value - start.Value;
-      if (length <= 0) {
-         return new List<T?>(0);
-      }
-      if (start.Value > 0) {
-         var nSkipped = checked((int)reader.Skip(start.Value));
-         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
-         if (nSkipped < start.Value) {
-            return new List<T?>(0);
-         }
-      }
-      int n = reader.ReadBatch(buffer!, 0, length);
-      return buffer.WrapSubRange(0, n);
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      reader.Dispose();
-   }
-}
-
-class ParquetColumnReader<TIn, TOut> : IParquetColumnReader<TOut> {
-   readonly Func<TIn, TOut?> convert;
-   readonly LogicalColumnReader<TIn> reader;
-   readonly TIn[] buffer;
-
-   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, int bufferSize)
-      : this(reader, convert, new TIn[bufferSize]) { }
-
-   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, TIn[] buffer) {
-      this.convert = convert;
-      this.reader = reader.LogicalReader<TIn>();
-      this.buffer = buffer;
-   }
-
-   ~ParquetColumnReader() =>
-      Dispose();
-
-   public IList<TOut?> Read() {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      int n = reader.ReadBatch(buffer!);
-      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
-   }
-
-   public IList<TOut?> Read(int? start, int? stop) {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      start ??= 0;
-      stop ??= buffer.Length;
-      int length = stop.Value - start.Value;
-      if (length <= 0) {
-         return new List<TOut?>(0);
-      }
-      if (start.Value > 0) {
-         var nSkipped = checked((int)reader.Skip(start.Value));
-         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
-         if (nSkipped < start.Value) {
-            return new List<TOut?>(0);
-         }
-      }
-      int n = reader.ReadBatch(buffer!, 0, length);
-      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      reader.Dispose();
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetExtensions.cs

@@ -1,159 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Microsoft.Scripting.Utils;
-
-using ParquetSharp;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-public static class ParquetExtensions {
-   public static int RowCount(this FileMetaData fileMetaData) =>
-      checked((int)fileMetaData.NumRows);
-
-   public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
-      var colInfos = new ColumnInfo[schemaDescriptor.NumColumns];
-      for (int i = 0; i < colInfos.Length; i++) {
-         colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata);
-      }
-      return new BlueTableSchema(tableName, colInfos);
-   }
-
-   public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") =>
-      ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata);
-
-   public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
-      var c = schemaDescriptor.Column(index);
-      var lt = c.LogicalType;
-      if (lt.Type == LogicalTypeEnum.Enum) {
-         var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name);
-         if (enumType is not null) {
-            return new(c.Name, index, enumType);
-         }
-      }
-      return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType));
-   }
-
-   public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) {
-      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      IBlueRowCollection? rv;
-      if (!schema.columnInfo.Any(c => c.colName == "id")) {
-         rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop));
-      } else {
-         rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop));
-      }
-      Log.assert(!schema.columnInfo.Any(c => c is null));
-      return rv;
-   }
-
-   public static (Column[] Column, IReadOnlyDictionary<string, string> Meta) ToParquetSchema(this IBlueTableSchema schema) {
-      var blueColumns = schema.columnInfo;
-      var parquetColumns = new Column[blueColumns.Length];
-      var parquetMeta = new Dictionary<string, string>();
-      KeyValuePair<string, string>[] parquetColMeta;
-      for (int i = 0; i < blueColumns.Length; i++) {
-         (parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]);
-         parquetMeta.AddRange(parquetColMeta);
-      }
-      return (parquetColumns, parquetMeta);
-   }
-
-   public static (Column Column, KeyValuePair<string, string>[] Meta) ToParquetColumn(this ColumnInfo columnInfo) {
-      var colType = columnInfo.colType;
-      bool isNullable = colType.IsNullable();
-      if (isNullable) {
-         colType = colType.GetGenericArguments()[0];
-      }
-      KeyValuePair<string, string>[] meta;
-      TypeCode typeCode;
-      var colName = columnInfo.colName;
-      if (colType.IsEnum) {
-         typeCode = colType.GetEnumUnderlyingType().GetTypeCode();
-         meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) };
-      } else {
-         typeCode = colType.GetTypeCode();
-         meta = Array.Empty<KeyValuePair<string, string>>();
-      }
-      return (ToParquetColumn(colName, typeCode), meta);
-
-      static Column ToParquetColumn(string colName, TypeCode typeCode) =>
-         typeCode switch {
-            TypeCode.Boolean => new Column<bool?>(colName),
-            TypeCode.Byte => new Column<byte?>(colName),
-            TypeCode.Int16 => new Column<short?>(colName),
-            TypeCode.Int32 => new Column<int?>(colName),
-            TypeCode.Int64 => new Column<long?>(colName),
-            TypeCode.UInt16 => new Column<ushort?>(colName),
-            TypeCode.UInt32 => new Column<uint?>(colName),
-            TypeCode.UInt64 => new Column<ulong?>(colName),
-            TypeCode.Single => new Column<float?>(colName),
-            TypeCode.Double => new Column<double?>(colName),
-            TypeCode.DateTime => new Column<DateTimeNanos?>(colName),
-            TypeCode.String => new Column<string?>(colName),
-            _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
-         };
-   }
-
-   static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) {
-      return logicalType.Type switch {
-         // LogicalTypeEnum.Int => TypeCode.Int32,
-         // LogicalTypeEnum.Decimal => TypeCode.Double,
-         LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime,
-         LogicalTypeEnum.String => TypeCode.String,
-         LogicalTypeEnum.Int => FromInt(),
-         LogicalTypeEnum.Decimal => FromDecimal(),
-         _ => FromPhysicalType(),
-      };
-
-      TypeCode FromPhysicalType() =>
-         physicalType switch {
-            PhysicalType.Boolean => TypeCode.Boolean,
-            PhysicalType.Int32 => TypeCode.Int32,
-            PhysicalType.Int64 => TypeCode.Int64,
-            PhysicalType.Float => TypeCode.Single,
-            PhysicalType.Double => TypeCode.Double,
-            _ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"),
-         };
-
-      TypeCode FromInt() {
-         var ilt = (IntLogicalType)logicalType;
-         var bitWidth = ilt.BitWidth;
-         return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt();
-
-         TypeCode FromSignedInt() =>
-            bitWidth switch {
-               8 => TypeCode.SByte,
-               16 => TypeCode.Int16,
-               32 => TypeCode.Int32,
-               64 => TypeCode.Int64,
-               _ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"),
-            };
-
-         TypeCode FromUnsignedInt() =>
-            bitWidth switch {
-               8 => TypeCode.Byte,
-               16 => TypeCode.UInt16,
-               32 => TypeCode.UInt32,
-               64 => TypeCode.UInt64,
-               _ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"),
-            };
-      }
-
-      TypeCode FromDecimal() {
-         var dlt = (DecimalLogicalType)logicalType;
-         var scale = dlt.Scale;
-         var precision = dlt.Precision;
-         return scale switch {
-            0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"),
-            _ => TypeCode.Double,
-         };
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetGroupTable.cs

@@ -1,234 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-public class ParquetGroupTable : IBlueRowCollection, IDisposable {
-   readonly RowGroupReader reader;
-   readonly IBlueTableSchema schema;
-   readonly bool leaveOpen;
-   readonly int fullCount;
-   readonly object syncRoot = new();
-   readonly int? start;
-   readonly int? stop;
-   readonly int startIndex;
-   readonly int stopIndex;
-   readonly int idColumnIndex;
-   readonly int idColumnIndexInFile;
-   readonly Func<int, int>? rowIdFactory;
-   DataCache? dataCacheBacking;
-
-   public ParquetGroupTable(
-      RowGroupReader reader,
-      IBlueTableSchema schema,
-      int? start = null,
-      int? stop = null,
-      int idColumnIndex = -1,
-      int idColumnIndexInFile = -1,
-      Func<int, int>? rowIdFactory = null,
-      bool leaveOpen = false
-   ) {
-      this.reader = reader;
-      this.schema = schema;
-      this.start = start;
-      this.stop = stop;
-      this.idColumnIndex = idColumnIndex;
-      this.idColumnIndexInFile = idColumnIndexInFile;
-      this.rowIdFactory = rowIdFactory;
-      this.leaveOpen = leaveOpen;
-      fullCount = checked((int)reader.MetaData.NumRows);
-      Log.assert(fullCount >= 0);
-      startIndex = SliceList.GetStartIndex(start, fullCount, true);
-      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
-      stopIndex = Math.Max(stopIndex, startIndex);
-      Log.assert(
-         startIndex >= 0
-         && startIndex <= fullCount
-         && stopIndex >= 0
-         && stopIndex <= fullCount
-         && stopIndex >= startIndex);
-   }
-
-   ~ParquetGroupTable() =>
-      Dispose();
-
-   public int Count => stopIndex - startIndex;
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   DataCache dataCache =>
-      EnsureInit();
-
-   public IBlueRow this[int id] => dataCache[id];
-
-   public bool containsID(int id) =>
-      dataCache.RowIdToIndex.ContainsKey(id);
-
-   public IBlueRow GetRow(int index) =>
-      dataCache.Rows[index];
-
-   public int IndexOf(IBlueRow item) {
-      int id = item.Id();
-      return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item)
-         ? idx
-         : -1;
-   }
-
-   public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) {
-      if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) {
-         row = dataCache.Rows[idx];
-         return true;
-      }
-      row = null;
-      return false;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      for (int i = 0; i < dataCache.Rows.Length; i++) {
-         yield return dataCache.Rows[i];
-      }
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      if (!leaveOpen) {
-         reader.Dispose();
-      }
-   }
-
-   [MemberNotNull(nameof(dataCacheBacking))]
-   internal DataCache EnsureInit() {
-      if (dataCacheBacking.HasValue) {
-         return dataCacheBacking.Value;
-      }
-      lock (syncRoot) {
-         if (dataCacheBacking.HasValue) {
-            return dataCacheBacking.Value;
-         }
-         var rows = new IBlueRow[Count];
-         int numColumns = reader.MetaData.NumColumns;
-         var blueColumns = schema.columnInfo;
-         Log.assert((blueColumns.Length - numColumns) switch {
-            0 or 1 => true,
-            _ => false,
-         });
-         var columnValues = allocateColumnValues(
-            reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory);
-         var rowIdToIndex = new Dictionary<int, int>();
-         for (int i = 0; i < rows.Length; i++) {
-            rows[i] = new BlueRowColumnWrapper(schema, i, columnValues);
-         }
-         dataCacheBacking = new(rows, rowIdToIndex, columnValues);
-      }
-      return dataCacheBacking.Value;
-   }
-
-   static IList<object?>[] allocateColumnValues(
-      RowGroupReader reader,
-      int numColumns,
-      int numRows,
-      int idColIdx,
-      int idColIdxInFile,
-      int startIndex,
-      int stopIndex,
-      Func<int, int>? idFactory
-   ) {
-      if (idColIdx < 0 || idColIdx >= numColumns) {
-         throw new ArgumentException("Id column index out of bounds", nameof(idColIdx));
-      }
-      var columnValues = new IList<object?>[numColumns];
-      if (idColIdxInFile == -1) {
-         // Generate ids (there is no id column in the file)
-         ArgumentNullException.ThrowIfNull(idFactory);
-         for (int i = 0; i < numColumns; i++) {
-            if (i < idColIdx) {
-               ReadColumn(i, i);
-            } else if (i == idColIdx) {
-               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
-            } else {
-               ReadColumn(i - 1, i);
-            }
-         }
-      } else if (idFactory is not null) {
-         // Generate ids (skip id column in the file)
-         for (int i = 0; i < numColumns; i++) {
-            if (i == idColIdx) {
-               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
-            } else {
-               if (i > idColIdx && i < idColIdxInFile) {
-                  ReadColumn(i - 1, i);
-               } else if (i > idColIdxInFile && i < idColIdx) {
-                  ReadColumn(i + 1, i);
-               } else {
-                  ReadColumn(i, i);
-               }
-            }
-         }
-      } else {
-         // Get ids from the file (output table id column index may differ from file)
-         // Case 1) File id col index > output
-         //    src cols: a, b, c, d, (id), e, f, ...
-         //    dst cols: a, b, (id), c, d, e, f, ...
-         //
-         // Case 2) File id col index == output
-         //    src cols: a, b, (id), c, d, e, f, ...
-         //    dst cols: a, b, (id), c, d, e, f, ...
-         //
-         // Case 3) File id col index < output
-         //    src cols: a, b, (id), c, d, e, f, ...
-         //    dst cols: a, b, c, d, (id), e, f, ...
-
-         for (int i = 0; i < numColumns; i++) {
-            if (i == idColIdx) {
-               ReadColumn(idColIdxInFile, idColIdx);
-            } else if (i > idColIdx && i < idColIdxInFile) {
-               ReadColumn(i - 1, i);
-            } else if (i > idColIdxInFile && i < idColIdx) {
-               ReadColumn(i, i - 1);
-            } else {
-               ReadColumn(i, i);
-            }
-         }
-      }
-      return columnValues;
-
-      void ReadColumn(int srcIdx, int dstIdx) {
-         using var colReader = reader.GetColumnReader(srcIdx);
-         columnValues[dstIdx] = colReader.Read(startIndex, stopIndex);
-      }
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-
-   internal readonly struct DataCache {
-      public readonly IBlueRow[] Rows;
-      public readonly Dictionary<int, int> RowIdToIndex;
-      public readonly IList<object?>[] ColumnValues;
-
-      public DataCache(IBlueRow[] rows, Dictionary<int, int> rowIdToIndex, IList<object?>[] columnValues) {
-         Rows = rows;
-         RowIdToIndex = rowIdToIndex;
-         ColumnValues = columnValues;
-      }
-
-      public IBlueRow this[int id] =>
-         Rows[RowIdToIndex[id]];
-
-      public IBlueRow GetRow(int index) =>
-         Rows[index];
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetReaderStream.cs

@@ -1,65 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Threading;
-
-[DataStreamUserRespresentation(false)]
-public class ParquetReaderStream : ViewStream {
-   readonly string file;
-   readonly int? start;
-   readonly int? stop;
-   readonly IWorkScheduler workScheduler;
-   IBlueRowCollection? table;
-   IDisposable? workerSub;
-
-   public ParquetReaderStream(string file, IWorkScheduler workScheduler) {
-      this.file = file;
-      this.workScheduler = workScheduler;
-   }
-
-   public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) {
-      this.file = file;
-      this.start = start;
-      this.stop = stop;
-      this.workScheduler = workScheduler;
-   }
-
-   protected override void myOnFirstSubscriber() {
-      Start();
-      var worker = new AnonymousWorker(DoWork);
-      workScheduler.Add(worker);
-      workerSub = Disposable.Create(() => workScheduler.Remove(worker));
-
-      async void Start() {
-         try {
-            Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop));
-         } catch (Exception ex) {
-            Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}");
-         }
-      }
-
-      void DoWork() {
-         var table = Volatile.Read(ref this.table);
-         if (table is null) {
-            return;
-         }
-         Disposable.dispose_thread_safe(ref workerSub);
-         if (table.schema.columnInfo.Length > 1) {
-            myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose_thread_safe(ref workerSub);
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetRow.cs

@@ -1,26 +0,0 @@
-// namespace Bts.DataStream.Parquet;
-
-// using BCG;
-
-// using global::Parquet.Data;
-
-// public class ParquetRow : IBlueRow {
-//    readonly int id;
-//    readonly int rowIndex;
-//    readonly DataColumn[] data;
-
-//    public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) {
-//       this.id = id;
-//       this.rowIndex = rowIndex;
-//       this.data = data;
-//       Schema = schema;
-//    }
-
-//    public object? this[int index] =>
-//       index == 0
-//          ? id
-//          : data[index - 1].Data.GetValue(rowIndex);
-
-//    public IBlueTableSchema Schema { get; }
-//    IBlueTableSchema IBlueRow.schema => Schema;
-// }

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTable.cs

@@ -1,145 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-
-public class ParquetTable : IBlueRowCollection, IDisposable {
-   readonly SortedList<int, ParquetGroupTable> groups = new();
-   readonly int fullCount;
-   readonly IBlueTableSchema schema;
-   readonly ParquetFileReader? reader;
-   readonly int startIndex;
-   readonly int stopIndex;
-
-   public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) {
-      schema = reader.FileMetaData.ToBlueSchema(tableName);
-      int idColIdx;
-      if (schema.TryResolveIndices("id", out int idColIdxInFile)) {
-         idColIdx = idColIdxInFile;
-      } else {
-         var origCols = schema.columnInfo;
-         var cols = new ColumnInfo[origCols.Length + 1];
-         idColIdx = 0;
-         cols[0] = new("id", 0, typeof(int));
-         for (int i = 0; i < origCols.Length; i++) {
-            var c = origCols[i];
-            cols[i + 1] = new(c.colName, i + 1, c.colType);
-         }
-         schema = new BlueTableSchema(cols, schema.tableName);
-      }
-      fullCount = checked((int)reader.FileMetaData.NumRows);
-      Log.assert(fullCount >= 0);
-      startIndex = SliceList.GetStartIndex(start, fullCount, true);
-      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
-      stopIndex = Math.Max(stopIndex, startIndex);
-      Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex);
-      int numRowGroups = reader.FileMetaData.NumRowGroups;
-      int rowGroupLastIdx = 0;
-      object syncRoot = new();
-      for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) {
-         int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null;
-         int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null;
-         int offset = rowGroupLastIdx;
-         var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId);
-         if (grp.Count == 0) {
-            continue;
-         }
-         rowGroupLastIdx += grp.Count;
-         groups.Add(rowGroupLastIdx, grp);
-
-         int GetAutoId(int index) =>
-            index + offset;
-      }
-      Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit());
-      if (!leaveOpen) {
-         this.reader = reader;
-      }
-   }
-
-   ~ParquetTable() =>
-      Dispose();
-
-   public IBlueTableSchema Schema => schema;
-
-   public int Count => stopIndex - startIndex;
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public IBlueRow this[int index] {
-      get {
-         foreach (var grp in groups.Values) {
-            if (grp.TryGetValue(index, out var r)) {
-               return r;
-            }
-         }
-         throw new KeyNotFoundException();
-      }
-   }
-
-   public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") =>
-      new(new(path), tableName, start, stop, false);
-
-   public IBlueRow GetRow(int index) {
-      if (index < 0 || index >= Count) {
-         throw new IndexOutOfRangeException();
-      }
-      int rowGroupIdx = groups.Keys.BinarySearch(index);
-      if (rowGroupIdx < 0) {
-         rowGroupIdx = ~rowGroupIdx;
-      }
-      Log.assert(rowGroupIdx <= groups.Count);
-      var grp = groups[rowGroupIdx];
-      int idxWithinGrp = index;
-      if (rowGroupIdx > 0) {
-         idxWithinGrp -= groups.Keys[rowGroupIdx - 1];
-      }
-      return grp[idxWithinGrp];
-   }
-
-   public bool containsID(int id) =>
-      groups.Values.Any(rgl => rgl.containsID(id));
-
-   public int IndexOf(IBlueRow item) {
-      int idx;
-      int grpFirstIdx = 0;
-      foreach (var (grpLastIdx, grp) in groups) {
-         idx = grp.IndexOf(item);
-         if (idx >= 0) {
-            return idx + grpFirstIdx;
-         }
-         grpFirstIdx = grpLastIdx;
-      }
-      return -1;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      foreach (var rgl in groups.Values) {
-         foreach (var r in rgl) {
-            yield return r;
-         }
-      }
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      groups.Values.DisposeAll();
-      reader?.Dispose();
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableReader.cs

@@ -1,32 +0,0 @@
-#nullable enable
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using System.IO;
-using System.Threading.Tasks;
-
-public class ParquetTableReader {
-   public static Task<IBlueRowCollection> ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) =>
-      Task.Run(() => Read(stream, leaveOpen, start, stop));
-
-   public static Task<IBlueRowCollection> ReadAsync(string path, int? start = null, int? stop = null) =>
-      Task.Run(() => Read(path, start, stop));
-
-   public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) {
-      using var fileReader = new ParquetFileReader(stream, leaveOpen);
-      var rv = fileReader.ReadBlueTable(start, stop);
-      rv.schema.Validate();
-      return rv;
-   }
-
-   public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) {
-      IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop);
-      rv.schema.Validate();
-      return rv;
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableWriter.cs

@@ -1,38 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using bts;
-
-using ParquetSharp;
-
-using System.IO;
-using System.Threading.Tasks;
-
-public static class ParquetTableWriter {
-   public static Task WriteAsync(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) =>
-      Task.Run(() => Write(blueRows, outStream, leaveOpen));
-
-   public static void Write(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) {
-      var (parquetColumns, parquetMeta) = blueRows.schema.ToParquetSchema();
-      using var schema = Column.CreateSchemaNode(parquetColumns);
-      using var writerProps = createWriterProperties();
-      using var writer = new ParquetFileWriter(outStream, schema, writerProps, keyValueMetadata: parquetMeta, leaveOpen: leaveOpen);
-      using var rowGroup = writer.AppendRowGroup();
-      for (int i = 0; i < blueRows.schema.columnInfo.Length; i++) {
-         var col = blueRows.schema.columnInfo[i];
-         Log.assert(col.colNo == i);
-         using var colWriter = ParquetAdapter.GetColumnWriter(col.type_code(), rowGroup.NextColumn());
-         colWriter.Write(blueRows.GetColumn(i));
-      }
-      writer.Close();
-   }
-
-   static WriterProperties createWriterProperties(Compression compression = Compression.Snappy) {
-      using var builder = new WriterPropertiesBuilder();
-      builder.Compression(compression);
-      return builder.Build();
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetWriterStream.cs

@@ -1,87 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts.collections;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-using Spark.Core.Threading;
-
-using System;
-using System.IO;
-
-[DataStreamUserRespresentation(false)]
-public class ParquetWriterStream : ViewStream {
-   readonly string file;
-   readonly ViewStream inputStream;
-   readonly IWorkScheduler workScheduler;
-   readonly AtomicGate gate = new();
-   IDataPacket? pendingOutput;
-   IDisposable? dispose;
-
-   public ParquetWriterStream(ViewStream inputStream, string file, IWorkScheduler workScheduler) {
-      this.inputStream = inputStream;
-      this.file = file;
-      this.workScheduler = workScheduler;
-   }
-
-   protected override void myOnFirstSubscriber() {
-      BlueTableImmutable.DataPacketBuilder? outputBuilder = null;
-      bool isFirstUpdate = false;
-      var worker = new AnonymousWorker(DoWork);
-      inputStream.onChange += OnInputStreamChange;
-      workScheduler.Add(worker);
-      dispose = Disposable.Create(Dispose);
-
-      void Dispose() {
-         workScheduler.Remove(worker);
-         inputStream.onChange -= OnInputStreamChange;
-      }
-
-      void OnInputStreamChange(object? dpObj) {
-         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dpObj);
-         using (gate.BeginTransaction()) {
-            outputBuilder ??= new BlueTableImmutable(dp.Schema).ToDataPacketBuilder();
-            foreach (IBlueRow r in dp.addedRows) {
-               outputBuilder.AddOrUpdate(r);
-            }
-            foreach (IBlueRow r in dp.deletedRows) {
-               outputBuilder.Remove(r.Id());
-            }
-         }
-      }
-
-      async void DoWork() {
-         using (gate.BeginTransaction()) {
-            if (outputBuilder is null) {
-               return;
-            }
-            if (pendingOutput is not null) {
-               myFireEvent(pendingOutput);
-               pendingOutput = null;
-            }
-            var dp = outputBuilder.ToDataPacket();
-            if (isFirstUpdate) {
-               isFirstUpdate = false;
-            } else if (!dp.HasChanges()) {
-               return;
-            }
-            {
-               using var outStream = File.OpenWrite(file);
-               await ParquetTableWriter.WriteAsync(dp.newRows, outStream);
-            }
-            // Fire the update at the beginning of the next doWork call so it happens on the
-            // 'work' thread.
-            pendingOutput = dp;
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose_thread_safe(ref dispose);
-}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,5 +1,8 @@
 namespace bts.BlueOrb {
    using BCG;
+
+   using Spark.Core.Diagnostics;
+
    using System;
    using System.Collections;
    using System.Collections.Generic;
@@ -54,7 +57,7 @@ namespace bts.BlueOrb {
 
       public bool IsReadOnly { get { return true; } }
 
-      public object this[object key] {
+      public object? this[object key] {
          get {
             if (key is string)
                return this[(string)key];
@@ -63,16 +66,15 @@ namespace bts.BlueOrb {
          set { throw new NotSupportedException("The IBlueRow is readonly"); }
       }
 
-      public object this[int index] {
+      public object? this[int index] {
          get {
             return this[schema.columnInfo[index].colName];
          }
       }
 
-      public object this[string col_name] {
+      public object? this[string col_name] {
          get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
                return _row[mapped_to_col];
             }
             return _row[col_name];
@@ -89,7 +91,8 @@ namespace bts.BlueOrb {
          }
       }
 
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Add(object key, object? value) =>
+         throw new NotSupportedException();
 
       public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
 
@@ -114,7 +117,7 @@ namespace bts.BlueOrb {
 
       private ViewStream _stream;
       private string _project_expression;
-      private ProjectData _project_data = null;
+      ProjectData? _project_data;
       //public ProjectData 
       public NewProjectStream(ViewStream vs, string project_expression) {
          _stream = vs;
@@ -128,9 +131,8 @@ namespace bts.BlueOrb {
          return _project_data.out_schema;
       }
 
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
+      public IBlueRow map(IBlueRow row) =>
+         new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
 
       // this is where most of the logic is, in terms of translating cols
       private static ProjectData _create_project_data(IBlueTableSchema input_schema,

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -1,18 +1,7 @@
-using BCG;
-
-using bts.utils;
-
-using Bts.DataStreamCore;
-
-using mbts.utils;
-
-using Spark.Core.Collections;
-using Spark.Core.Threading;
-
 using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-using System.Linq;
+using System.Data.Entity.Core.Mapping;
+
+using BCG;
 
 namespace bts.BlueOrb {
    // Keeps adding changing rows to the stream, hence preserving the history
@@ -20,20 +9,17 @@ namespace bts.BlueOrb {
    // into a time series of prices/qty
    [DataStreamUserRespresentation("TimeSeriesStream")]
    public class TimeSeriesStream : ViewStream {
-      readonly ViewStream _stream;
-      readonly string timeColumn;
-      readonly AtomicGate gate = new();
-      CircularBufferTable? _table;
+      private CircularBufferTable _table;
+      private ViewStream _stream;
       private int _id = 0;
       private int _size = 0;
 
       public TimeSeriesStream(ViewStream stream)
          : this(stream, Int32.MaxValue) { }
 
-      public TimeSeriesStream(ViewStream stream, int size, string timeColumn = "ts_event") {
+      public TimeSeriesStream(ViewStream stream, int size) {
          _stream = stream;
          _size = size;
-         this.timeColumn = timeColumn;
       }
 
       protected override void myOnFirstSubscriber() {
@@ -45,37 +31,27 @@ namespace bts.BlueOrb {
 
 
       private void _onstream_change(object o) {
-         var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
+         DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
-         using var _ = gate.BeginTransaction();
          if (_table == null) {
-            var colInfos = dp.newRows.schema.columnInfo;
-            if (colInfos.TryGetFirst(c => c.colName == timeColumn, out var currTimeCol, out var currTimeColIdx)) {
-               colInfos = colInfos.ToArray();
-               colInfos[currTimeColIdx] = new(timeColumn, currTimeCol.colNo, typeof(DateTime));
-            } else {
-               colInfos = ArrayHelper.add(colInfos, new(timeColumn, colInfos.Length, typeof(DateTime)));
-            }
-            _table = new CircularBufferTable(new BlueTableSchema(colInfos, dp.newRows.schema.tableName), _size);
+            IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
+            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            _table = new CircularBufferTable(schema, _size);
          }
-         var addedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
-         var deletedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue(timeColumn, now);
-            int id = _id++;
-            ar.setValue("id", id);
-            _table = _table.Add(ar, out var oldRow);
-            addedRowsBd.AddOrUpdate(ar);
-            if (oldRow is not null) {
-               deletedRowsBd.AddOrUpdate(oldRow);
-            }
+            ar.setValue("timestamp", now);
+            ar.setValue("id", _id++);
+            _table.add_row(ar);
          }
-         var addedRows = addedRowsBd.ToImmutable();
-         var deletedRows = deletedRowsBd.ToImmutable();
-         var dp_out = new DataPacket(_table, addedRows, deletedRows);
+         var dp_out =  BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
          myFireEvent(dp_out);
       }
    }
+
+
+
 }
 
+
+

src/DataStreamSuite/DataStream/BlueOrb/TimeseriesIndex.cs

@@ -1,197 +0,0 @@
-namespace Bts.DataStream;
-
-using bts;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-
-/// <summary>
-/// Maps a timeseries of keys to unique int32 ids.
-/// </summary>
-public class TimeseriesIndex<T> where T : notnull {
-   static readonly IComparer<KeyValuePair<DateTime, int>> idTimeseriesComp =
-      KeyValuePairComparer.KeyComparer<DateTime, int>();
-
-   readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId;
-   readonly ImmutableDictionary<int, TimeseriesKey<T>> idToName;
-   readonly int idCounter;
-
-   public TimeseriesIndex()
-      : this(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Empty, ImmutableDictionary<int, TimeseriesKey<T>>.Empty, 0) { }
-
-   TimeseriesIndex(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
-      this.nameToId = nameToId;
-      this.idToName = idToName;
-      this.idCounter = idCounter;
-   }
-
-   public int Count => idToName.Count;
-
-   public ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> NameToId => nameToId;
-   public ImmutableDictionary<int, TimeseriesKey<T>> IdToName => idToName;
-
-   public bool ContainsId(int id) =>
-      idToName.ContainsKey(id);
-
-   public int GetNearest(TimeseriesKey<T> key) {
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         throw new KeyNotFoundException();
-      }
-      int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-      if (idx < 0) {
-         idx = ~idx;
-         if (idx == 0) {
-            throw new KeyNotFoundException();
-         }
-         idx--;
-      }
-      return ids[idx].Value;
-   }
-
-   public bool TryGetNearest(TimeseriesKey<T> key, out int result) {
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         result = -1;
-         return false;
-      }
-      int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-      if (idx < 0) {
-         idx = ~idx;
-         if (idx == 0) {
-            result = -1;
-            return false;
-         }
-         idx--;
-      }
-      result = ids[idx].Value;
-      return true;
-   }
-
-   public TimeseriesIndex<T> GetNearestOrAdd(TimeseriesKey<T> key, out int result) {
-      int idx;
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-         idx = 0;
-      } else {
-         idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-         if (idx < 0) {
-            idx = ~idx;
-         }
-      }
-      if (idx == 0) {
-         result = idCounter + 1;
-         ids = ids.Insert(0, new(key.Timestamp, result));
-         return new(
-            nameToId.SetItem(key.Name, ids),
-            idToName.SetItem(result, key),
-            result);
-      }
-      result = ids[idx].Value;
-      return this;
-   }
-
-   public TimeseriesIndex<T> GetOrAdd(TimeseriesKey<T> key, out int result) {
-      int idx;
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-         idx = 0;
-      } else {
-         idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-         if (idx < 0) {
-            idx = ~idx;
-         } else {
-            result = ids[idx].Value;
-            return this;
-         }
-      }
-      result = idCounter + 1;
-      ids = ids.Insert(idx, new(key.Timestamp, result));
-      return new(
-         nameToId.SetItem(key.Name, ids),
-         idToName.SetItem(result, key),
-         result);
-   }
-
-   public IList<KeyValuePair<DateTime, int>> GetIds(T name) =>
-      nameToId[name];
-
-   public Builder ToBuilder() =>
-      new(this);
-
-   public class Builder {
-      readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Builder nameToId;
-      readonly ImmutableDictionary<int, TimeseriesKey<T>>.Builder idToName;
-      int idCounter;
-
-      public Builder(TimeseriesIndex<T> source)
-         : this(source.nameToId, source.idToName, source.idCounter) { }
-
-      public Builder(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
-         this.nameToId = nameToId.ToBuilder();
-         this.idToName = idToName.ToBuilder();
-         this.idCounter = idCounter;
-      }
-
-      public int GetNearest(TimeseriesKey<T> key) =>
-         getOrAdd(key, false, false);
-
-      public int GetNearestOrAdd(TimeseriesKey<T> key) =>
-         getOrAdd(key, false, true);
-
-      public int GetOrAdd(TimeseriesKey<T> key) =>
-         getOrAdd(key, true, true);
-
-      public int GetLatest(T key) =>
-         nameToId.TryGetValue(key, out var ids)
-            ? ids[^1].Value
-            : throw new KeyNotFoundException();
-
-      public bool TryGetLatest(T key, out int result) {
-         if (nameToId.TryGetValue(key, out var ids)) {
-            result = ids[^1].Value;
-            return true;
-         }
-         result = 0;
-         return false;
-      }
-
-      public ImmutableList<KeyValuePair<DateTime, int>> GetIds(T name) =>
-         nameToId.TryGetValue(name, out var ids)
-            ? ids
-            : ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-
-      public TimeseriesIndex<T> ToImmutable() =>
-         new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter);
-
-      int getOrAdd(TimeseriesKey<T> key, bool requireExactTimestamp, bool canAdd) {
-         var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList<KeyValuePair<DateTime, int>>.Empty);
-         int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp);
-         int id;
-         if (idx < 0) {
-            Log.Gen.info($"Adding key to timeseries index ({key.Timestamp}, {key.Name})");
-            // We didn't get an exact match. If requireExactTimestamp is false and a record with an
-            // earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp.
-            idx = ~idx;
-            if (requireExactTimestamp || idx == 0) {
-               if (!canAdd) {
-                  throw new KeyNotFoundException();
-               }
-               id = ++idCounter;
-               ids = ids.Insert(idx, new(key.Timestamp, id));
-               nameToId[key.Name] = ids;
-               idToName.Add(id, key);
-            } else {
-               if (idx == ids.Count) {
-                  --idx;
-               }
-               id = ids[idx].Value;
-            }
-         } else {
-            id = ids[idx].Value;
-         }
-         return id;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TimeseriesKey.cs

@@ -1,22 +0,0 @@
-namespace Bts.DataStream;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-
-public readonly record struct TimeseriesKey<T>(T Name, DateTime Timestamp) : IDualComparable<TimeseriesKey<T>> {
-   public int CompareTo(TimeseriesKey<T> other) {
-      var rv = Timestamp.CompareTo(other.Timestamp);
-      if (rv == 0) {
-         rv = Comparer<T>.Default.Compare(Name, other.Name);
-      }
-      return rv;
-   }
-
-   public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey<T> key) =>
-      (key.Name, key.Timestamp);
-
-   public static implicit operator TimeseriesKey<T>((T Name, DateTime Timestamp) tup) =>
-      new(tup.Name, tup.Timestamp);
-}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -5,7 +5,8 @@ namespace bts.BlueOrb {
    using mbts;
    using BCG;
 
-   [DataStreamUserRespresentation("UnionStream")]
+   // [DataStreamUserRespresentation("UnionStream")]
+   [DataStreamUserRespresentation("AppendStream")]
    public class UnionStream : ViewStream {
       public UnionStream(ViewStream left_stream, ViewStream right_stream) {
          ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -6,24 +6,37 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
-    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="ParquetSharp" Version="20.0.0" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+        <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
+        <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+        <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+        <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
+        <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ArithmeticExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="BaseUtilities" Version="$(SparkVersion)" />
+        <PackageReference Include="BooleanExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.ColumnExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.ColumnSortExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="DataStreamExpressions" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
+
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,7 +2,6 @@ namespace Bts.DataStream;
 
 using BCG;
 
-using bts;
 using bts.api.core;
 
 using Bts.DataStreamCore;
@@ -16,13 +15,27 @@ using Spark.Core.Threading;
 using System;
 using System.Linq;
 using System.Reactive.Linq;
-using System.Threading;
-using System.Threading.Tasks;
 
 public static class DataStreamExtensions {
+   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
+      source as ViewStream ?? new ObservableViewStream(source);
+
    public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
       source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
 
+   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<IDataPacket> observer) {
+         source.onChange += OnChange;
+         return Disposable.Create(() => source.onChange -= OnChange);
+
+         void OnChange(object? newValue) =>
+            observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
+               $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
+      }
+   }
+
    // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
    //    source.Select(dp => {
    //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
@@ -46,16 +59,16 @@ public static class DataStreamExtensions {
    public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
       this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
       rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
-         var bd = res.to_builder();
+      return source.Scan(TableUpdate<T, TKey>.Empty(keySelector), (res, dp) => {
+         var bd = res.ToBuilder();
          dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
          foreach (IBlueRow r in addedRows) {
-            bd.add_or_update(valueSelector(r));
+            bd.AddOrUpdate(valueSelector(r));
          }
          foreach (IBlueRow r in deletedRows) {
-            bd.remove(rowKeySelector(r));
+            bd.Remove(rowKeySelector(r));
          }
-         return bd.to_table();
+         return bd.ToImmutable();
       });
    }
 
@@ -65,16 +78,16 @@ public static class DataStreamExtensions {
          (res, table) => {
             var resTable = res.Table;
             if (resTable is null) {
-               if (table.state.Count == 0) {
+               if (table.Index.Count == 0) {
                   return (null, null);
                }
-               resTable = new(table.state.Values.First().schema);
+               resTable = new(table.Index.Values.First().schema);
             }
             var bd = resTable.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
+            foreach (var r in table.RemovedItems) {
                bd.Remove(r.Id());
             }
-            foreach (var r in table.added) {
+            foreach (var r in table.AddedItems) {
                bd.AddOrUpdate(r);
             }
             var resDp = bd.ToDataPacket(out var newRows);
@@ -88,10 +101,10 @@ public static class DataStreamExtensions {
          (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
          (res, table) => {
             var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
+            foreach (var r in table.RemovedItems) {
                bd.Remove(r.Id());
             }
-            foreach (var r in table.added) {
+            foreach (var r in table.AddedItems) {
                bd.AddOrUpdate(r);
             }
             var resDp = bd.ToDataPacket(out var newRows);
@@ -121,6 +134,46 @@ public static class DataStreamExtensions {
    public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
       source.AsDataPackets(schema).AsViewStream();
 
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
+      var columns = schema.columnInfo;
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
+         }
+      }
+      return indices;
+   }
+
    public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
       Observable.Create<IDataPacket>(o => {
          var inputGate = new AtomicGate();
@@ -145,7 +198,7 @@ public static class DataStreamExtensions {
                foreach (IBlueRow r in value.addedRows) {
                   bd.AddOrUpdate(r);
                }
-               pendingDpOut = bd.ToDataPacket(out lastTable, false);
+               pendingDpOut = bd.ToDataPacket(out lastTable);
             }
          }
 
@@ -161,118 +214,5 @@ public static class DataStreamExtensions {
          }
       });
 
-   public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
-      const int STATE_IDLE = 0;
-      const int STATE_BUSY = 1;
-      const int STATE_DISPOSED = 2;
-      return Observable.Create<IDataPacket>(Subscribe);
-      IDisposable Subscribe(IObserver<IDataPacket> o) {
-         var syncRoot = new object();
-         IBlueRowCollection? lastTableIn = null;
-         BlueTableImmutable? lastTableOut = null;
-         int nConcurrentDoWorks = 0;
-         int state = STATE_IDLE;
-         IDisposable? dispose = null;
-         dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
-         return Disposable.Create(Dispose);
 
-         void Dispose() {
-            lock (syncRoot) {
-               switch (state) {
-                  case STATE_DISPOSED:
-                     return;
-                  case STATE_BUSY:
-                     state = STATE_DISPOSED;
-                     return;
-               }
-               state = STATE_DISPOSED;
-            }
-            Disposable.dispose(ref dispose);
-         }
-
-         void OnNext(IDataPacket value) =>
-            Volatile.Write(ref lastTableIn, value.newRows);
-
-         async void DoWork() {
-            lock (syncRoot) {
-               nConcurrentDoWorks++;
-               switch (state) {
-                  case STATE_DISPOSED:
-                  case STATE_BUSY:
-                     return;
-                  case STATE_IDLE:
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
-                     return;
-               }
-               state = STATE_BUSY;
-            }
-            var dp = await PublishTable();
-            bool isDisposed = false;
-            lock (syncRoot) {
-               switch (state) {
-                  case STATE_DISPOSED:
-                     isDisposed = true;
-                     break;
-                  case STATE_BUSY:
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
-                     return;
-               }
-            }
-            if (isDisposed) {
-               Disposable.dispose(ref dispose);
-               return;
-            }
-            if (dp is not null) {
-               o.OnNext(dp);
-            }
-            lock (syncRoot) {
-               nConcurrentDoWorks = 0;
-               switch (state) {
-                  case STATE_DISPOSED:
-                     isDisposed = true;
-                     break;
-                  case STATE_BUSY:
-                     state = STATE_IDLE;
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
-                     return;
-               }
-            }
-            if (isDisposed) {
-               Disposable.dispose(ref dispose);
-            }
-
-            async Task<DataPacket?> PublishTable() {
-               var tableIn = Interlocked.Exchange(ref lastTableIn, null);
-               if (tableIn is null) {
-                  return null;
-               }
-               var tableOut = await transformAsync(tableIn);
-               if (tableOut is null) {
-                  return null;
-               }
-               lastTableOut ??= new(tableOut.schema);
-               var bd = lastTableOut.ToDataPacketBuilder();
-               bd.Clear();
-               foreach (IBlueRow r in tableOut) {
-                  bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
-               }
-               return bd.ToDataPacket(out lastTableOut);
-            }
-         }
-
-         static string StateName(int state) =>
-            state switch {
-               STATE_IDLE => "IDLE",
-               STATE_BUSY => "BUSY",
-               STATE_DISPOSED => "DISPOSED",
-               _ => $"STATE_{state}",
-            };
-      }
-   }
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -3,27 +3,30 @@ namespace bts.BlueOrb {
    using BCG;
    using System.Collections.Generic;
    using Bts.DataStreamExpressions;
-   using Bts.DataStreamCore;
+   using System.Xml;
 
    // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
    // So for example, if an expresison is passed 
    [DataStreamUserRespresentation("dsl")]
    public class DSLStream : ViewStream {
       private readonly string _dsl;
-      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
+      public DSLStream(string dsl) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         // Remove all colons
+         string sanitizedDsl = dsl.Replace(":", "COLON");
          _dsl = dsl;
-         var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
          mySetRealSource(rs);
       }
 
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
+      static ViewStream? build_stream(string stream_expression) {
          try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
          } catch (Exception ex) {
+            //send to blueorb
             Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,14 +1,13 @@
 namespace Bts.DataStream;
 
 using BCG;
-using BCG.Extensions;
 using BCG.Utilities.General;
 
 using bts;
-using bts.collections;
 
 using Bts.DataStreamCore;
 
+using Spark.Core.Collections;
 using Spark.Core.Disposables;
 
 using System;

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -40,7 +40,7 @@ namespace BCG {
         protected override void myOnFirstSubscriber() {
             MasterWorker.getInstance(false).addWorker(this, _send_period);
             var server_rci = new RemoteConnectionInfo(_ip, _port);
-            bts.AsyncTCPConnector_AsyncWrapper.connect_to(server_rci, this);
+            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
         }
 
         protected override void myOnLastSubscriber() {
@@ -76,4 +76,4 @@ namespace BCG {
 
 
     }
-}
+}

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -1,5 +1,4 @@
-#nullable enable
-namespace Bts.DataStreamCore;
+namespace Bts.DataStream;
 
 using BCG;
 

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -1,5 +1,6 @@
 using bts;
 
+using Bts.DataStream;
 using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
@@ -11,7 +12,6 @@ using Spark.Core.Disposables;
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Reactive.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -35,12 +35,12 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
+      static ViewStream? build_stream(string stream_expression)
       {
          try
          {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
          } catch (Exception ex) {
@@ -52,85 +52,43 @@ namespace BCG {
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       readonly IPubTableBroker _broker;
       readonly IManagedSubscriber _sub;
-      readonly IDataStreamFactory streamFactory;
-      readonly ITableStreamFactory tableStreamFactory;
       readonly object syncRoot = new();
       IDisposable? cloudStreamsSub;
       IDisposable? remoteRequestsSub;
-      State state;
+      bool isDisposed;
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, ITableStreamFactory tableStreamFactory, IEnumerable<IDataStreamFactory> streamFactories) {
+
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
          _sub = managedSubscriber;
          _broker = pubTableBroker;
-         this.tableStreamFactory = tableStreamFactory;
-         var streamFactoryList = streamFactories.ToList();
-         streamFactory = streamFactoryList.Count == 1
-            ? streamFactoryList[0]
-            : new CompositeDataStreamFactory(streamFactoryList);
-      }
-
-      enum State {
-         Idle = 0,
-         Running = 1,
-         Disposed = 2,
-         WaitingToDispose = 3,
       }
 
       public Task StartAsync(CancellationToken cancellationToken) {
          lock (syncRoot) {
-            switch (state) {
-               case State.Idle:
-                  break;
-               case State.WaitingToDispose:
-               case State.Disposed:
-                  throw new ObjectDisposedException(nameof(RemoteQueryManager));
-               default:
-                  return Task.CompletedTask;
-            }
-            state = State.Running;
-         }
-         if (cloudStreamsSub is not null) {
-            throw new InvalidOperationException("cloudStreamsSub already created");
-         }
-         if (remoteRequestsSub is not null) {
-            throw new InvalidOperationException("remoteRequestsSub already created");
-         }
-         var cloudStream = tableStreamFactory.Create("cloud_streams")
-            ?? throw new Exception("Failed to create TableStream 'cloud_stream'");
-         cloudStreamsSub = cloudStream.AsObservable().Subscribe(_on_str);
-         remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
-         lock (syncRoot) {
-            if (state == State.WaitingToDispose) {
-               state = State.Disposed;
-            }
-         }
-         if (state == State.Disposed) {
-            Disposable.dispose(ref cloudStreamsSub);
-            Disposable.dispose(ref remoteRequestsSub);
+            Log.assert(!isDisposed);
+            Log.isnull(cloudStreamsSub);
+            Log.isnull(remoteRequestsSub);
+            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
+            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
+               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+            });
          }
          return Task.CompletedTask;
       }
 
       public Task StopAsync(CancellationToken cancellationToken) {
          lock (syncRoot) {
-            switch (state) {
-               case State.Running:
-                  break;
-               default:
-                  return Task.CompletedTask;
+            if (!isDisposed) {
+               isDisposed = true;
+               Disposable.dispose(ref cloudStreamsSub);
+               Disposable.dispose(ref remoteRequestsSub);
             }
-            state = State.Disposed;
          }
-         Disposable.dispose(ref cloudStreamsSub);
-         Disposable.dispose(ref remoteRequestsSub);
          return Task.CompletedTask;
       }
 
-      private void _on_str(IDataPacket dp)
-      {
+      void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
@@ -144,11 +102,7 @@ namespace BCG {
 
       private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         var stream = build_stream(streamFactory, stream_expression);
-         if (stream is null) {
-            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
-            return;
-         }
+         ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
          if (persist) {
             string tempDir = Path.GetTempPath();

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -2,30 +2,17 @@ using BCG;
 
 using bts;
 
-using Bts.DataStreamCore;
-
-using Spark.Core.Disposables;
-
 using System;
 
 namespace BCG {
-   /// <summary>
-   /// This is a stream so we can run it from the DSQA (since dsqa can only load streams).
-   /// <para>
-   /// Note: It's hidden because this stream is now created by an IDataStreamFactory
-   /// (see RemoteQueryStreamFactory.cs for details)
-   /// </para>
-   /// </summary>
-   [DataStreamUserRespresentation(false)]
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
    public class RemoteQueryStream : ViewStream {
       private readonly string _name, _stream_exp;
-      readonly ITableStreamFactory tableStreamFactory;
-      readonly IPublisherCore _publisher_core;
-      IDisposable? subscription;
+      private PublisherCore _publisher_core = PublisherCore.Create();
+      private ViewStream _source_str;
 
-      public RemoteQueryStream(ITableStreamFactory tableStreamFactory, IPublisherCore publisherCore, params string[] lines) {
-         this.tableStreamFactory = tableStreamFactory;
-         _publisher_core = publisherCore;
+      public RemoteQueryStream(params string[] lines) {
          _stream_exp = get_expression(lines);
          _name = Guid.NewGuid().ToString();
          _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
@@ -34,17 +21,17 @@ namespace BCG {
       }
 
       protected override void myOnFirstSubscriber() {
-         Log.isnull(subscription);
          RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
          byte[] rem_req_bytes = rem_req.serialize();
          _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
-         var sourceStream = tableStreamFactory.Create(_name);
-         sourceStream.onChange += _handle_data;
-         subscription = Disposable.Create(() => sourceStream.onChange -= _handle_data);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
       }
 
-      protected override void myOnLastSubscriber() =>
-         Disposable.dispose(ref subscription);
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }
 
       static string get_expression(params string[] lines) {
          for (int i = 0; i < lines.Length; i++) {

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -2,9 +2,6 @@ namespace bts {
    using System.IO;
    using BCG;
 
-   using System.IO;
-   using System.Threading.Tasks.Dataflow;
-
    public class RemoteRequest {
       public readonly string stream_expression;
       public readonly string published_table_name;
@@ -28,4 +25,4 @@ namespace bts {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,33 +1,28 @@
+using Spark.Core.Collections;
+
 using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace BCG {   
    public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_single_tok(string exp) {
+      public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
          string[] toks = exp.Split(':');
          string col_name = toks[0].Trim();
          string col_value = toks[1].Trim();
          Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return Tuple.Create(col_name, tup.Item1, tup.Item2);
+         return (col_name, tup.Item1, tup.Item2);
       }
 
-      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
-         string[] toks = exp.Split(",");
-         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
-         foreach (string s in toks) {
-            var tup = parse_single_tok(s.Trim());
-            rv.Add(tup);
-         }
-         return rv;
-      }
+      public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
+         exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
 
-      private Tuple<string, Type, object> _new_col;
-      private List<Tuple<string, Type, object>> _new_col_map;
-      private IBlueTableSchema _out_schema;
+      readonly List<(string Name, Type Type, object Value)> _new_col_map;
+      IBlueTableSchema? _out_schema;
 
       public AddColumnsStream(ViewStream vs, string col_def) {
          _new_col_map = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
+         ViewStream rs = new MapStream(vs, _get_schema, _map);
          mySetRealSource(rs);
       }
 
@@ -44,7 +39,7 @@ namespace BCG {
       private IBlueRow _map(IBlueRow r) {
          MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
          foreach (var m in _new_col_map) {
-            rv.setValue(m.Item1, m.Item3);
+            rv.setValue(m.Name, m.Value);
          }
          return rv;
       }

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs

@@ -1,9 +1,10 @@
-using System;
+using mbts;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using mbts;
 
 namespace BCG
 {
@@ -33,7 +34,7 @@ namespace BCG
 
          private void _stream_onChange(object newValue)
          {
-            DataPacket dp = newValue as DataPacket;
+            var dp = (IDataPacket)newValue;
             if (dp != null && dp.newRows.Count > 0)
             {
                _stream.onChange -= _stream_onChange;
@@ -42,8 +43,8 @@ namespace BCG
          }
       }
 
-      public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
-      public delegate string GetColumnPrefix(object[] transposing_col_values);
+      public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
+      public delegate string GetColumnPrefix(object?[] transposing_col_values);
 
       public delegate string GetColumnSuffix(string col_name);
       private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
@@ -51,33 +52,31 @@ namespace BCG
       private IncrementValues _increment_values;
       private GetColumnPrefix _get_column_prefix;
 
-      private IBlueTableSchema _schema;
+      IBlueTableSchema? _schema;
 
       private string[] _group_by_cols;
       private Pair<string, Type>[] _cols_per_transposed;
       private string[] _transposing_cols;
 
-      private int[] _transposing_col_indices;
+      int[]? _transposing_col_indices;
 
-      private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
+      Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
          : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
                 get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
       { }
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
       {
          if (get_column_suffix_del != null)
             _get_column_suffix = get_column_suffix_del;
          if (transposed_col_value_comparer != null)
          {
-            _transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
-               {
-                  return transposed_col_value_comparer(p1.second, p2.second);
-               });
+            _transpsed_col_value_comparer_wrapper = new((p1, p2) =>
+               transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
          }
          _group_by_cols = group_by_cols.Split(' ');
          _transposing_cols = transposed_cols_group.Split(' ');
@@ -103,10 +102,10 @@ namespace BCG
          List<ColumnInfo> col_info = new List<ColumnInfo>();
          col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
          foreach (Pair<string, Type> p in group_by_col_info)
-            col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
+            col_info.Add(new ColumnInfo(p.first ?? string.Empty, col_index++, ExceptionAssert.NotNull(p.second)));
          foreach (string tc_prefix in tc_prefixes)
             foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, ExceptionAssert.NotNull(p.second)));
          IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
          if (_schema == null)
             _schema = schema;
@@ -130,19 +129,14 @@ namespace BCG
 
       public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
       {
-         Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
-         Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
+         var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
+         var group_by_cols_value_dict = new Dictionary<int, object?[]>();
          BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
          Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
          string table_name = string.Empty;
          if (grouped_rows.Count > 0)
          {
-            IBlueTableSchema schema = null;
-            foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
-            {
-               schema = kvp.Value[0].schema;
-               break;
-            }
+            var schema = grouped_rows.Values.First()[0].schema;
             table_name = schema.tableName;
             _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
             int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
@@ -155,23 +149,22 @@ namespace BCG
          else
             return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); 
          Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
-         List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
+         var tc_values = new List<Pair<int, object?[]>>();
          foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
          {
             //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values 
-            Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
+            var incrementing_values = new Dictionary<int, object?[]>();
             foreach (IBlueRow r in kvp.Value)
             {
                int hash = hasher.get_hash(r, _transposing_col_indices);
-               object[] values;
-               if (!incrementing_values.TryGetValue(hash, out values))
+               if (!incrementing_values.TryGetValue(hash, out var values))
                {
                   incrementing_values.Add(hash, _get_per_transposing_col_values(r));
                   string prefix = _get_column_prefix(_get_transposing_col_values(r));
                   if (!tc_prefixes.ContainsKey(hash))
                   {
                      tc_prefixes.Add(hash, prefix);
-                     tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
+                     tc_values.Add(new(hash, _get_transposing_col_values(r)));
                   }
                }
                else
@@ -189,23 +182,23 @@ namespace BCG
          if (_transpsed_col_value_comparer_wrapper != null)
             tc_values.Sort(_transpsed_col_value_comparer_wrapper);
          List<string> prefixes = new List<string>();
-         foreach (Pair<int, object[]> p in tc_values)
+         foreach (Pair<int, object?[]> p in tc_values)
             prefixes.Add(tc_prefixes[p.first]);
          _set_schema(prefixes, group_by_col_info, table_name);
          BlueTable result = new BlueTable(_schema);
          int id = 1;
-         foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
+         foreach (var kvp in all_tc_infos_and_values)
          {
             MutableBlueRow mbr = new MutableBlueRow(_schema);
             mbr.setValue("id", id++);
             for (int i = 0; i < _group_by_cols.Length; i++)
                mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
-            foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
+            foreach (var tc_cols in kvp.Value)
             {
                string prefix = tc_prefixes[tc_cols.Key];
                for (int i = 0; i < _cols_per_transposed.Length; i++)
                {
-                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
+                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
                   if(mbr.schema.columnNames.Contains(column))
                      mbr.setValue(column, tc_cols.Value[i]);
                }
@@ -217,28 +210,25 @@ namespace BCG
 
       #endregion
 
-      private object[] _get_transposing_col_values(IBlueRow row)
-      {
-         object[] values = new object[_transposing_cols.Length];
+      object?[] _get_transposing_col_values(IBlueRow row) {
+         var values = new object?[_transposing_cols.Length];
          for (int i = 0; i < _transposing_cols.Length; i++)
             values[i] = row[_transposing_cols[i]];
          return values;
       }
 
-      private object[] _get_per_transposing_col_values(IBlueRow row)
-      {
-         object[] values = new object[_cols_per_transposed.Length];
+      object?[] _get_per_transposing_col_values(IBlueRow row) {
+         var values = new object?[_cols_per_transposed.Length];
          for (int i = 0; i < _cols_per_transposed.Length; i++)
-            values[i] = row[_cols_per_transposed[i].first];
+            values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
          return values;
       }
 
-      private object[] _get_group_by_cols_values(IBlueRow row)
-      {
-         object[] values = new object[_group_by_cols.Length];
+      object?[] _get_group_by_cols_values(IBlueRow row) {
+         var values = new object?[_group_by_cols.Length];
          for (int i = 0; i < values.Length; i++)
             values[i] = row[_group_by_cols[i]];
          return values;
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsMapper.cs

@@ -1,17 +1,17 @@
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections;
+using System.Diagnostics.CodeAnalysis;
 using System.Text;
 
-using BCG;
-using BCG.Utilities;
-
 namespace BCG
 {
    public class ConcatenateColumnsMapper : IStreamMapper
    {
       protected IDictionary myTempColMap = new SortedList ();
-      protected ArrayList myColMap;
-      protected BlueTableSchema mySchema;
+      protected ArrayList? myColMap;
+      protected BlueTableSchema? mySchema;
 
       public ConcatenateColumnsMapper(params string [] columnDefs)
       {
@@ -55,11 +55,7 @@ namespace BCG
             myColMap = new ArrayList (myTempColMap.Count);
             foreach (DictionaryEntry de in myTempColMap)
             {
-               string[] cols = de.Value as string[];
-               ArrayList temp = new ArrayList ();
-               foreach (string col in cols)
-                  temp.Add (inputSchema[col].colNo);
-               myColMap.Add (temp.ToArray(typeof(int)) as int[]);
+               myColMap.Add(Array.ConvertAll((string[])de.Value!, c => inputSchema[c].colNo));
             }
             myTempColMap.Clear ();
          }
@@ -72,6 +68,8 @@ namespace BCG
 
       public virtual IBlueRow map (IBlueRow row)
       {
+         requireSchema();
+         requireColMap();
          object[] retVal = new object[mySchema.columnInfo.Length];
          row.CopyTo (retVal, myColMap.Count);
          for (int colNo = 0; colNo < myColMap.Count; colNo ++)
@@ -90,11 +88,21 @@ namespace BCG
       /// <returns></returns>
       protected virtual object myGetConcatValue (IBlueRow row, int colNo)
       {
+         requireColMap();
          StringBuilder b = new StringBuilder();
-         foreach (int cn in myColMap[colNo] as int[])
-            b.Append(row[cn].ToString());
+         foreach (int cn in (int[])myColMap[colNo]!) {
+            b.Append(row[cn]?.ToString());
+         }
          return b.ToString ();
       }
+
+      [MemberNotNull(nameof(myColMap))]
+      protected void requireColMap() =>
+         ExceptionAssert.NotNull(myColMap);
+
+      [MemberNotNull(nameof(mySchema))]
+      void requireSchema() =>
+         ExceptionAssert.NotNull(mySchema);
    }
 
    public class ConcatenateColumnsMapperForSameDataSet : ConcatenateColumnsMapper 
@@ -107,23 +115,21 @@ namespace BCG
          myDicts = new HashTableND [columnDefs.Length];
          int i = 0;
          foreach (DictionaryEntry de in myTempColMap)
-            myDicts[i++] = new HashTableND ((de.Value as string[]).Length);
+            myDicts[i++] = new HashTableND(((string[])de.Value!).Length);
       }
 
       protected override object myGetConcatValue (IBlueRow row, int colNo)
       {
-         int[] cols = myColMap[colNo] as int[];
-         object[] vals = new object[cols.Length];
-         for (int i = 0; i < vals.Length; i++)
-            vals[i] = row[cols[i]];
+         requireColMap();
          HashTableND ht = myDicts[colNo];
-         object rv = ht.getValue (vals);
+         var vals = Array.ConvertAll((int[])myColMap[colNo]!, c => row[c]);
+         var rv = ht.getValue(vals);
          if (rv != null)
             return rv;
          else
          {
             rv = base.myGetConcatValue (row, colNo);
-            object[] args = new object[vals.Length + 1];
+            var args = new object?[vals.Length + 1];
             vals.CopyTo (args, 0);
             args[args.Length-1] = rv;
             ht.add (args);

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -2,7 +2,7 @@ using System;
 
 namespace bts.BlueOrb {
    using BCG;
-   [DataStreamUserRespresentation("oldCount")]
+   [DataStreamUserRespresentation("CountStream")]
    public class CountStream : ViewStream {
       private DataPacket myLastFiredDataPacket;
       private IBlueTableSchema mySchema = new BlueTableSchema("count",
@@ -15,6 +15,9 @@ namespace bts.BlueOrb {
          myVS = vs;
       }
 
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
+
       protected override void myOnFirstSubscriber() {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -1,22 +1,19 @@
-using System;
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 
 namespace BCG
 {
    public class DerivativeStream : ViewStream 
    {
-
-      private IBlueTableSchema mySchema;
+      IBlueTableSchema? mySchema;
       private string myDerivandColumn;
       private string myOutputColumn;
       private string myKeyColumn;
       private DateTime myLastPacketArrival = DateTime.MinValue;      
       private ViewStream myDataSource;
-      
-      private IDictionary<string, double> myPreviousValues;
-
+      Dictionary<string, double>? myPreviousValues;
 
       public DerivativeStream(ViewStream input, string keyColumn,  string columnToDifferentiate, string outputColName)
       {
@@ -39,9 +36,9 @@ namespace BCG
 
       void myDataSource_onChange(object newValue)
       {
-         DataPacket dp = (DataPacket)newValue;
+         var dp = (IDataPacket)newValue;
          if (myLastPacketArrival == DateTime.MinValue)
-            mySchema = myBuildOutputSchema(dp.schema);
+            mySchema = myBuildOutputSchema(dp.Schema);
          else
          {
             DateTime now = DateTime.Now;
@@ -55,7 +52,7 @@ namespace BCG
             else
             {
                IBlueRowCollection deleted, added;
-               BlueDataUtility.computeDifferences(thisVersion, ((DataPacket)myLastValue).newRows, out deleted, out added);
+               BlueDataUtility.computeDifferences(thisVersion, ((IDataPacket)myLastValue).newRows, out deleted, out added);
                myFireEvent(new DataPacket(thisVersion, added, deleted));  
             }
          }
@@ -64,12 +61,13 @@ namespace BCG
 
       protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
       {
+         ExceptionAssert.NotNull(mySchema, myPreviousValues);
          object[] outValues = new object[mySchema.columnInfo.Length];
          r.Values.CopyTo(outValues, 0);
 
-         object derivand = r[myDerivandColumn];
-         double lastValue = default(double);
-         if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value)
+         var derivand = r[myDerivandColumn];
+         var key = getRowKey(r);
+         if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
             outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
          else
             outValues[mySchema.columnInfo.Length - 1] = 0.0;
@@ -84,16 +82,19 @@ namespace BCG
          cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double));
          return new BlueTableSchema("derviative of " + inSchema.tableName, cols);
       }
-      protected void myStoreNewPacketInfo(DataPacket dp)
-      {
+
+      protected void myStoreNewPacketInfo(IDataPacket dp) {
          myPreviousValues = new Dictionary<string, double>();
          myLastPacketArrival = DateTime.Now;
          foreach (IBlueRow row in dp.newRows)
          {
-            object derivand = row[myDerivandColumn];
+            var derivand = row[myDerivandColumn];
             if (derivand != null && derivand != DBNull.Value)
-               myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand);
+               myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
          }
       }
+
+      string getRowKey(IBlueRow row) =>
+         row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
    }
 }

src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs

@@ -1,7 +1,9 @@
-using System;
-using System.Collections;
+using mbts;
+
+using Spark.Core.Collections;
+
+using System;
 using System.Collections.Generic;
-using mbts;
 
 namespace BCG
 {
@@ -12,9 +14,9 @@ namespace BCG
    {
       ViewStream myLStream, myRStream;
 
-      DataPacket myLastR = null, myLastL = null;
-
-      BlueTable myLastFired = null;
+      DataPacket? myLastR;
+      DataPacket? myLastL;
+      BlueTable? myLastFired;
 
       double myThreshold = .00005;
 
@@ -24,14 +26,14 @@ namespace BCG
       public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : 
          this(null, v1, v2, thresh, ColumnsToMatch) { }
 
-      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
       {
          myThreshold = thresh;
          myLStream = v1;
          myRStream = v2;
          myColumnsToMatch = columnsToMatch;
       }
-      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
          : this(db,v1,v2,.00005,columnsToMatch)
       {
       }
@@ -50,45 +52,22 @@ namespace BCG
          myRStream.onChange -= myRChange;
       }
 
-      struct ListComparer : IEqualityComparer<List<object>>
-      {
-         public bool Equals(List<object> x, List<object> y)
-         {
-            if (x.Count != y.Count)
-               return false;
-
-            for (int i = 0; i < x.Count; ++i)
-               if (!x[i].Equals(y[i]))
-                  return false;
-            return true;
-         }
-
-         public int GetHashCode(List<object> obj)
-         {
-            int ret = 0;
-            for(int i = 0; i < obj.Count; ++i)
-               ret ^= obj[i].GetHashCode();
-            return ret;
-         }
-      }
-
       void myHandleCallbacks()
       {
-         DataPacket packet = null;
-
+         DataPacket? packet;
          if (myLastL == null || myLastR == null)
          {
             packet = myLastL == null ? myLastR : myLastL;
-            if (packet != null)
+            if (packet is not null) {
                myFireEvent(packet);
-
-            myLastFired = new BlueTable(packet.newRows);
+               myLastFired = new BlueTable(packet.newRows);
+            }
             return;
          }
 
          BlueTable table = new BlueTable(myLastL.newRows.schema);
 
-         Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
+         var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
 
          foreach(BlueRow row in myLastL.newRows)
          {
@@ -118,11 +97,7 @@ namespace BCG
                }
                catch (Exception) { }
             }
-
-            Pair<BlueRow, BlueRow> pair;
-            rowPairs.TryGetValue(hash, out pair);
-
-
+            rowPairs.TryGetValue(hash, out var pair);
             if (pair == null)
                pair = new Pair<BlueRow, BlueRow>(null, row);
             else
@@ -138,8 +113,7 @@ namespace BCG
 
             if (pair.first == null || pair.second == null)
             {
-               BlueRow row = pair.first == null ? pair.second : pair.first;
-
+               var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
                Dictionary<string, object> newRow = new Dictionary<string, object>();
                foreach (ColumnInfo info in row.schema.columnInfo)
                {
@@ -211,4 +185,4 @@ namespace BCG
 
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs

@@ -5,7 +5,7 @@ namespace BCG
 {
 	public class FileViewStream : ViewStream
 	{
-      private DataPacket myDp;
+      DataPacket? myDp;
 		public FileViewStream (string dataFileName, 
          string schemaFileName)
          : this (dataFileName, schemaFileName, Int32.MaxValue)

src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs

@@ -1,8 +1,7 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
+
 namespace BCG
 {
     public class DateBasedFilter: IDynamicStreamFilter, IWorker
@@ -26,20 +25,14 @@ namespace BCG
             myReference = myTodaysDate.AddDays(-1 * myNDays);                     
         }       
         
-        public event FilterChangeDelegate onFilterChange;       
+        public event FilterChangeDelegate? onFilterChange;       
         
-        public bool filter(IBlueRow r)
-        {
-            if (DBNull.Value.Equals(r[myDateColumn]))
-            {
-                //return false;
-                return true; //it can be considered as our assumption  whenever datecolumn is db null we return true 
-            }
-            DateTime t = (DateTime)r[myDateColumn];                   
-            bool returnVallue =  t < myReference ? false : true;
-            return returnVallue;
-        }
-              
+        public bool filter(IBlueRow r) =>
+           r[myDateColumn] switch {
+              DBNull or null => true,
+              DateTime t => t >= myReference,
+              _ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
+           };
 
         public void doWork()
         {
@@ -47,7 +40,7 @@ namespace BCG
             {
                 myTodaysDate = DateTime.Today;
                 myReference = myTodaysDate.AddDays(-1 * myNDays);
-                onFilterChange();
+                onFilterChange?.Invoke();
             }         
         }      
     }

src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs

@@ -1,8 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
 
 namespace BCG
 {
@@ -26,7 +24,7 @@ namespace BCG
 
       #region IDynamicStreamFilter Members
 
-      public event FilterChangeDelegate onFilterChange;
+      public event FilterChangeDelegate? onFilterChange;
 
       #endregion
 
@@ -34,7 +32,10 @@ namespace BCG
 
       public bool filter(IBlueRow r)
       {
-         DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date;
+         DateTime rowDate = r[myDateColName] switch {
+            DBNull or null => DateTime.Now.Date,
+            object obj => (DateTime)obj,
+         };
          DateTime today = DateTime.Now.Date;
          return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
       }
@@ -48,7 +49,7 @@ namespace BCG
          if (DateTime.Now.Date != myLastCheckedTime.Date)
          {
             myLastCheckedTime = DateTime.Now;
-            onFilterChange();
+            onFilterChange?.Invoke();
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,18 +1,23 @@
 using bts;
 
-using Bts.DataStream;
 using Bts.DataStream.Primitives.Filter;
+using Bts.DataStreamCore;
 
-using mbts.config;
+using Spark.Core.Diagnostics;
+using Spark.Core.Disposables;
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
 using System.Text.RegularExpressions;
 
 namespace BCG.StreamWorld.Addition {
    [DataStreamUserRespresentation(false)]
    public class FilterStream : ViewStream {
       private ViewStream myBaseStream;
-      private BlueTable myBaseTable, myStateTable;
+      BlueTable? myBaseTable;
+      BlueTable? myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
@@ -25,8 +30,8 @@ namespace BCG.StreamWorld.Addition {
          myBaseStream = vs;
          myFilter = filter;
          vs.ErrorChanged += OnError;
-         if (filter is IDynamicStreamFilter)
-            (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         if (filter is IDynamicStreamFilter dfilter)
+            dfilter.onFilterChange += myHandleFilterChange;
       }
 
 
@@ -36,8 +41,8 @@ namespace BCG.StreamWorld.Addition {
 
       protected override void myOnLastSubscriber() {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
-         myBaseTable.clear();
-         myStateTable.clear();
+         myBaseTable?.clear();
+         myStateTable?.clear();
       }
 
       bool filterRow(IBlueRow r) {
@@ -52,17 +57,14 @@ namespace BCG.StreamWorld.Addition {
 
       private void myHandleChanges(object newValue) {
          try {
-            DataPacket dp = newValue as DataPacket;
-            if (myStateTable == null) {
-               bts.Log.assert(myBaseTable == null);
-               myStateTable = new BlueTable(dp.schema);
-               myBaseTable = new BlueTable(dp.schema);
-            }
-            BlueTable dels = new BlueTable(dp.schema),
-                        adds = new BlueTable(dp.schema);
+            var dp = (IDataPacket)newValue;
+            var schema = dp.Schema;
+            init(schema);
+            var dels = new BlueTable(schema);
+            var adds = new BlueTable(schema);
             foreach (IBlueRow r in dp.deletedRows) {
                myBaseTable.deleteRow(r);
-               if (myStateTable.containsID((int)r["id"])) {
+               if (myStateTable.containsID(r.Id())) {
                   dels.addRow(r);
                   myStateTable.deleteRow(r);
                }
@@ -85,6 +87,7 @@ namespace BCG.StreamWorld.Addition {
          set {
             myFilter = value;
             if (myBaseTable != null) {
+               ExceptionAssert.NotNull(myStateTable);
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
                // If the filter throws an exception, we can rethrow it as a StreamFilterException
@@ -112,6 +115,17 @@ namespace BCG.StreamWorld.Addition {
       //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
 
       private void myHandleFilterChange() { filter = myFilter; }
+
+      [MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
+      void init(IBlueTableSchema schema) {
+         if (myStateTable is null) {
+            Log.isnull(myBaseTable);
+            myStateTable = new BlueTable(schema);
+            myBaseTable = new BlueTable(schema);
+         } else {
+            Log.notnull(myBaseTable);
+         }
+      }
    }
 }
 
@@ -135,29 +149,24 @@ namespace BCG
       }
 
       #region IStreamFilter Members
-      public bool filter(IBlueRow dr)
-      {
-         bool rv = true;
-         for (int i = 0; i < myCols.Length; i++)
-         {
-            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-            if (!rv)
-               return rv;
-         }
-         return rv;
-      }
+      public bool filter(IBlueRow dr) =>
+         myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
+            DBNull or null => string.Empty,
+            object obj => obj.ToString() ?? string.Empty
+         }));
       #endregion
    }
 
    public class FilterStream : ViewStream
    {
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
-      private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
+      BlueTable? myLastFiredRows;
+      IBlueRowCollection? myLastReceivedRows;
+      IStreamFilter? myFilter;
+      readonly ViewStream? myParentStream;
+      string? myFilterString;
+      readonly StreamWorld.Addition.FilterStream? myFilterStream;
+      readonly bool myUseNewFilter = false;
+      IDisposable? subscription;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -187,9 +196,10 @@ namespace BCG
          {
             myParentStream = vs;
             myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+            myFilterString = filter.ToString() ?? string.Empty;
+            if (filter is IDynamicStreamFilter dfilter) {
+               dfilter.onFilterChange += myHandleFilterChange;
+            }
          }
       }
 
@@ -213,18 +223,20 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber()
-      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() {
+         ExceptionAssert.IsNull(subscription);
+         subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
+      }
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         Disposable.Dispose(ref subscription);
       }
 
       protected override void myFireEvent(object o)
       {
-         DataPacket dp = o as DataPacket;
+         var dp = (IDataPacket)o;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -233,29 +245,30 @@ namespace BCG
       private void myHandleCallbacks(object newValue)
       {
          DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
+         var arrivedPacket = (IDataPacket)newValue;
          myLastReceivedRows = arrivedPacket.newRows;
          try {
             if (myLastFiredRows != null)
             {
-               retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-                  new BlueTable(myLastFiredRows.schema));
+               var addedRows = new BlueTable(myLastFiredRows.schema);
+               var deletedRows = new BlueTable(myLastFiredRows.schema);
+               retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
                foreach (IBlueRow r in arrivedPacket.deletedRows)
-                  if (myLastFiredRows.containsID((int)r["id"]))
+                  if (myLastFiredRows.containsID(r.Id()))
                   {
                      myLastFiredRows.deleteRow(r);
-                     (retVal.deletedRows as BlueTable).addRow(r);
+                     deletedRows.addRow(r);
                   }
                foreach (IBlueRow r in arrivedPacket.addedRows)
                   if (filterRow(r))
                   {
                      myLastFiredRows.addRow(r);
-                     (retVal.addedRows as BlueTable).addRow(r);
+                     addedRows.addRow(r);
                   }
             }
             else
             {
-               myLastFiredRows = new BlueTable(arrivedPacket.schema);
+               myLastFiredRows = new BlueTable(arrivedPacket.Schema);
                retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
                foreach (IBlueRow r in arrivedPacket.newRows)
                   if (filterRow(r))
@@ -273,7 +286,7 @@ namespace BCG
          {
             if (myUseNewFilter)
             {
-               myFilterStream.filter = filter;
+               requireNewFilter().filter = filter;
             }
             else
             {
@@ -302,9 +315,9 @@ namespace BCG
          get
          {
             if (myUseNewFilter)
-               return myFilterStream.filter;
+               return requireNewFilter().filter;
             else
-               return myFilter;
+               return requireOldFilter();
          }
       }
 
@@ -317,17 +330,25 @@ namespace BCG
 
       private void myHandleFilterChange()
       {
-         filter = myFilter;
+         filter = requireOldFilter();
       }
 
       bool filterRow(IBlueRow r) {
          try {
-            return myFilter.filter(r);
+            return requireOldFilter().filter(r);
          } catch (Exception ex) {
             Log.Gen.error($"FilterStream delegate threw exception: {ex}");
             OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
          }
          return false;
       }
+
+      [MemberNotNull(nameof(myFilterStream))]
+      StreamWorld.Addition.FilterStream requireNewFilter() =>
+         ExceptionAssert.MemberNotNull(myFilterStream);
+
+      [MemberNotNull(nameof(myFilter))]
+      IStreamFilter requireOldFilter() =>
+         ExceptionAssert.MemberNotNull(myFilter);
    }
 }

src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs

@@ -1,8 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
 
 namespace BCG
 {    
@@ -29,7 +27,7 @@ namespace BCG
             myReference = myCurrentTime.AddSeconds(-1 * myNSecs);                  
         }
         
-        public event FilterChangeDelegate onFilterChange;       
+        public event FilterChangeDelegate? onFilterChange;       
         
         public bool filter(IBlueRow r)
         {
@@ -38,9 +36,8 @@ namespace BCG
                 //it can be considered as our assumption  whenever datecolumn is db null we return true 
                 return true;  //dont use this for volparamsched use true              
             }
-            DateTime t = (DateTime)r[myDateColumn];                   
-            bool returnVallue =  t < myReference ? false : true;
-            return returnVallue;
+            var t = (DateTime)r[myDateColumn]!;
+            return t >= myReference;
         }
               
 
@@ -51,8 +48,7 @@ namespace BCG
                 myCurrentTime = DateTime.Now;
                 myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
                 myLastTimeFilteredApplied = DateTime.Now;              
-                onFilterChange();
-                
+                onFilterChange?.Invoke();
             }           
         }
     }

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -67,7 +67,7 @@ namespace BCG {
       }
       #endregion
 
-      public event FilterChangeDelegate onFilterChange;
+      public event FilterChangeDelegate? onFilterChange;
 
       #region IStreamFilter
       public bool filter(IBlueRow row) {

src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs

@@ -1,11 +1,13 @@
-using System;
+using BCG.Utilities.General;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
 
 using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
-using System.Diagnostics;
 
 namespace BCG
 {
@@ -40,12 +42,12 @@ namespace BCG
       private IComparer<IBlueRow> myRightCompare;
       protected FuzzyRowComparer myFuzzyCompare;
       protected TimeSpan myTimeWindow;
-      protected IBlueTableSchema myLeftSchema;
-      protected IBlueTableSchema myRightSchema;
-      protected IBlueTableSchema myOutputSchema;
+      protected IBlueTableSchema? myLeftSchema;
+      protected IBlueTableSchema? myRightSchema;
+      protected IBlueTableSchema? myOutputSchema;
       private int myID = 0;
 
-      protected TimeWindowTable myStateRows;
+      protected TimeWindowTable? myStateRows;
 
       private DateTime myNextCull;
       const long MinCullIntervalMS = 1000;
@@ -65,7 +67,7 @@ namespace BCG
          this.myFuzzyCompare = fuzzyCompare;
       }
 
-      public virtual IBlueTableSchema OutputSchema
+      public virtual IBlueTableSchema? OutputSchema
       {
          get
          {
@@ -150,7 +152,7 @@ namespace BCG
          int count;
          for (count = 0; count < list.Count; count++)
          {
-            if ((DateTime)list[count][timeColumn] > expiry)
+            if ((DateTime)list[count][timeColumn]! > expiry)
             {
                // Stop when we find the first row that we can keep.  Since this is sorted all others are also kept.
                break;
@@ -172,16 +174,16 @@ namespace BCG
             this.myFitness = fitness;
          }
          public IBlueRow myRightRow { get; private set; }
-         public IBlueRow myLeftRow { get; set; }
+         public IBlueRow? myLeftRow { get; set; }
          public double myFitness { get; private set; }
-         public IBlueRow myMergedRow { get; set; }
+         public IBlueRow? myMergedRow { get; set; }
 
          #region IComparable<RowMatch> Members
 
-         public int CompareTo(RowMatch other)
-         {
-            return myFitness.CompareTo(other.myFitness);
-         }
+         public int CompareTo(RowMatch? other) =>
+            other is null
+               ? 1
+               : myFitness.CompareTo(other.myFitness);
 
          #endregion
       }
@@ -262,12 +264,12 @@ namespace BCG
 
       private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
       {
+         ExceptionAssert.NotNull(myStateRows);
          // We already emitted correction.MergedRow so we need to redact that and 
          // then emit correction.left as an add since it is no longer matched
          foreach (RowMatch correction in corrections.Distinct())
          {
-            Debug.Assert(correction.myMergedRow != null);
-            this.myStateRows.RemoveRow(correction.myMergedRow);
+            myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow));
             adds.addRow(this.MergeRows(correction.myLeftRow, null));
             dels.addRow(correction.myMergedRow);
          }
@@ -285,7 +287,7 @@ namespace BCG
          int r = 0;
          for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
          {
-            if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
+            if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!)
             {
                // Advance Left
                IEnumerable<RowMatch> rm = rightMatches[l];
@@ -332,7 +334,7 @@ namespace BCG
 
       private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
       {
-         IBlueRow merged = null;
+         IBlueRow? merged = null;
          if (rightMatches != null)
          {
             foreach (RowMatch match in rightMatches)
@@ -366,9 +368,9 @@ namespace BCG
       /// Override this function to provide a custom row merge.
       /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
       /// </summary>
-      protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
+      protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
       {
-         Dictionary<string, object> newRow = new Dictionary<string, object>();
+         Dictionary<string, object?> newRow = new();
 
          if (left == null && right == null) throw new ArgumentException("oh come on.");
 
@@ -393,14 +395,15 @@ namespace BCG
             }
          }
          newRow["id"] = this.myID++;
-         newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
-
+         newRow[NEW_TIME_COLUMN] = left is not null
+            ? left[myLeftTimeColumn]
+            : right![myRightTimeColumn];
          return new BlueRow(this.OutputSchema, newRow);
       }
 
       private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
       {
-         List<RowMatch> matches = null;
+         List<RowMatch>? matches = null;
 
          // Search through all rows in list and compare with the fuzzy comparer
          for (int x = 0; x < list.Count; x++)
@@ -421,8 +424,7 @@ namespace BCG
                matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
 
                // Mark the best match for this particular right hand match
-               RowMatch oldMatch;
-               if (this.myBestMatch.TryGetValue(match, out oldMatch))
+               if (this.myBestMatch.TryGetValue(match, out var oldMatch))
                {
                   if (currentMatch.myFitness < oldMatch.myFitness)
                   {
@@ -441,7 +443,7 @@ namespace BCG
             }
          }
 
-         return matches;
+         return matches ?? Enumerable.Empty<RowMatch>();
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -1,4 +1,4 @@
-using System;
+using Bts.DataStreamCore;
 
 namespace BCG
 {
@@ -10,8 +10,8 @@ namespace BCG
 	{
       protected string[] myGroupByCols;
       protected string myOutputColumn;
-      protected GroupByOutputTable myOutputTable;
-      protected string myColMapExpression;
+      protected GroupByOutputTable? myOutputTable;
+      protected string? myColMapExpression;
 
       #region constructors
 		public GroupByEvaluator (string outputCol, string[] groupByCols)
@@ -52,31 +52,23 @@ namespace BCG
       /// <param name="r"></param>
       /// <param name="id"></param>
       /// <returns></returns>
-      protected IBlueRow myProject (IBlueRow r, int id)
-      {
-         return r == null ? null :
-            myColMapExpression == null || myColMapExpression.Length < 1 ?
-               BlueDataUtility.getNewReplacedRow (r, id, "id") :
-               BlueDataUtility.getNewReplacedRow(myProjectRow (r, myColMapExpression),
-            id, "id");
-      }
-      
+      protected IBlueRow? myProject(IBlueRow r, int id) =>
+         r is null
+            ? null
+            : myColMapExpression is null || myColMapExpression.Length < 1
+               ? BlueDataUtility.getNewReplacedRow(r, id, "id")
+               : BlueDataUtility.getNewReplacedRow(myProjectRow(r, myColMapExpression), id, "id");
+
       /// <summary>
       /// Projects the id, conserving the id
       /// </summary>
       /// <param name="r"></param>
       /// <returns></returns>
-      protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
+      protected IBlueRow? myProject(IBlueRow r) =>
+         myProject(r, r is null ? int.MinValue : r.Id());
       #endregion
 
-      private IBlueRow myProjectRow (IBlueRow r, string colMapExpression)
-      {
-         BlueTable auxTable = new BlueTable (r.schema);
-         auxTable.addRow (r);
-         BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable);
-         ProjectStream projStream = new ProjectStream (staticStream, colMapExpression);
-         return BlueDataUtility.extractRow (projStream.rows, 0);
-      }
-
+      IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
+         new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project"));
 	}
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs

@@ -1,5 +1,6 @@
+using Bts.DataStreamCore;
+
 using System;
-using BCG.Utilities;
 using System.Collections;
 
 namespace BCG
@@ -22,10 +23,12 @@ namespace BCG
       /// <param name="groupRepString"></param>
       public void addRow (IBlueRow row, string groupRepString)
       {
-         if (myDict[groupRepString] != null)
-            myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
+         var grp = myDict[groupRepString];
+         if (grp is not null) {
+            myTable.deleteRow((IBlueRow)grp);
+         }
          myTable.addRow (row);
-         myDict[groupRepString] = myTable[(int)row["id"]];
+         myDict[groupRepString] = myTable[row.Id()];
       }
 
       /// <summary>
@@ -35,7 +38,7 @@ namespace BCG
       public void removeRow (string groupRepString)
       {
          // BlueEventLog.logAssert (myDict[groupRepString] != null);
-         myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
+         myTable.deleteRow ((IBlueRow)myDict[groupRepString]!);
          myDict.Remove (groupRepString);
       }
 
@@ -48,11 +51,8 @@ namespace BCG
          myDict.Clear ();
       }
 
-      public IBlueRow getRow (string groupRepString)  
-      {
-         return myDict[groupRepString] == null ? null : 
-            myDict[groupRepString] as IBlueRow;
-      }
+      public IBlueRow? getRow(string groupRepString) =>
+         myDict[groupRepString] as IBlueRow;
 
       #region IBlueRowCollection members
       public IBlueRow this[int id]  {get{return myTable[id];}}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -1,7 +1,9 @@
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections;
-using BCG.Utilities;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 
 namespace BCG {
@@ -25,11 +27,12 @@ namespace BCG {
       /// Serves as  a quick way to access all rows belonging to the same group
       /// </summary>
       private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable;
+      RowDictionaryTable? myBookKeepingTable;
       private IGroupByEvaluator myGroupByEvaluator;
-      private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
+      // input schema, as output schema can differ (IGroupByEvaluator's
+      IBlueTableSchema? mySchema;
       //can map a schema to another
-      private IBlueTableSchema myOutputSchema;
+      IBlueTableSchema? myOutputSchema;
       private int myIdGenerator = 0;
 
       public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
@@ -45,14 +48,14 @@ namespace BCG {
       }
 
       protected override void myHandleFirstCall() {
+         requireLastArrivedPacket();
          mySchema = myLastArrivedPacket.schema;
          myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
          myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
             myGroupByEvaluator.groupByColumns);
          GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
          foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
-            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
-               rowGroup, new BlueTable(mySchema)));
+            var newRow = myEvaluate(new DataPacket(rowGroup, rowGroup, new BlueTable(mySchema)));
             if (newRow != null)
                temp.addRow(newRow, myGetGroupString(rowGroup));
          }
@@ -78,15 +81,14 @@ namespace BCG {
          ArrayList handledAddGroups = new ArrayList();
          string groupRepString;
 
+         requireBookKeeping();
          foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
             // BlueEventLog.logAssert (delGroup.Count > 0);
             groupRepString = myGetGroupString(delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
-               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) ?? new BlueTable(mySchema);
             handledAddGroups.Add(addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
-               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
-            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) ?? new BlueTable(mySchema);
+            var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
             myFireSwitch |= myMergeRow(newRow, groupRepString);
          }
 
@@ -98,16 +100,16 @@ namespace BCG {
                //the loop above
 
                groupRepString = myGetGroupString(addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
+               var aggGroup = ExceptionAssert.NotNull(myBookKeepingTable.getGroup(groupRepString));
                // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
-                  new BlueTable(aggGroup.schema)));
+               var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, new BlueTable(aggGroup.schema)));
                myFireSwitch |= myMergeRow(newRow, groupRepString);
             }
          }
       }
 
-      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
+      private bool myMergeRow(IBlueRow? newRow, string groupRepString) {
+         requireLastFiredPacket();
          IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
          //BlueEventLog.logAssert (newRow != null || existingRow != null);
          if (existingRow == null && newRow == null)
@@ -115,6 +117,7 @@ namespace BCG {
          else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
             return false;
          else if (existingRow == null) {
+            ArgumentNullException.ThrowIfNull(newRow);
             //row added for the first time
             ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
             ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
@@ -137,6 +140,9 @@ namespace BCG {
       /// <param name="delGroups"></param>
       /// <param name="addGroups"></param>
       private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
+         ExceptionAssert.NotNull(myBookKeepingTable);
+         requireLastFiredPacket();
+         requireLastArrivedPacket();
          delGroups =
             new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
          addGroups =
@@ -150,16 +156,19 @@ namespace BCG {
       }
 
       protected override void myOnTableClear() {
-         if (myBookKeepingTable.Count == 0)
-            return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
+         if (ExceptionAssert.NotNull(myBookKeepingTable).Count == 0) {
+            // ie there arent any rows, so no need to optimize the 'cleansing' process
+            return;  
+         }
+         requireLastFiredPacket();
          myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
          myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
+         ((GroupByOutputTable)myLastFiredPacket.newRows).clear();
          myBookKeepingTable.clear();
          myGroupByEvaluator.clear();
       }
 
-      private IBlueRow myEvaluate(DataPacket dp) {
+      private IBlueRow? myEvaluate(DataPacket dp) {
          IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
          return evalRow == null ? null :
             myIdGuard ?
@@ -191,7 +200,7 @@ namespace BCG {
          return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow(DataPacket dp) {
+      public static IBlueRow? getRepRow(DataPacket dp) {
          if (dp.newRows.Count > 0)
             return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)
@@ -210,13 +219,13 @@ namespace BCG {
       /// <param name="newValue"></param>
       /// <param name="isAdd"></param>
       /// <returns></returns>
-      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
+      public static object? incrementalAdd(object lastValue, object newValue, bool isAdd) {
          if (lastValue == null || lastValue.Equals(DBNull.Value))
             return newValue;
          // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
          // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
          // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
-         Object retVal = null;
+         object? retVal = null;
          if (newValue.GetType().Equals(typeof(int)))
             retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
          else if (newValue.GetType().Equals(typeof(Double)))
@@ -228,6 +237,9 @@ namespace BCG {
          return retVal;
       }
 
+      [MemberNotNull(nameof(myBookKeepingTable))]
+      void requireBookKeeping() =>
+         ExceptionAssert.NotNull(myBookKeepingTable);
    }
 
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MethodDrivenStream.cs

@@ -1,10 +1,9 @@
-using System;
-using System.Collections;
-using BCG.Utilities;
-using System.IO;
-
 namespace BCG
 {
+   using Spark.Core.Diagnostics;
+
+   using System.Diagnostics.CodeAnalysis;
+
    /// <summary>
    /// Some basic event handling functionality factorred out of any Stream whose behaviour
    /// is driven by some interface passed on by the user
@@ -12,8 +11,8 @@ namespace BCG
    public abstract class MethodDrivenStream : ViewStream
    {
       protected ViewStream myStream;
-      protected DataPacket myLastArrivedPacket;
-      protected DataPacket myLastFiredPacket;
+      protected DataPacket? myLastArrivedPacket;
+      protected DataPacket? myLastFiredPacket;
       protected bool myFireSwitch = true;
 
       public MethodDrivenStream(ViewStream vs)  {myStream = vs;}
@@ -59,6 +58,7 @@ namespace BCG
       /// To handle the first call, the LastFiredPacket is supposed to be initialized and filled
       /// with data.
       /// </summary>
+      [MemberNotNull(nameof(myLastFiredPacket))]
       protected abstract void myHandleFirstCall ();
 
       /// <summary>
@@ -89,5 +89,13 @@ namespace BCG
       {
          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
       }
+
+      [MemberNotNull(nameof(myLastArrivedPacket))]
+      protected void requireLastArrivedPacket() =>
+         ExceptionAssert.NotNull(myLastArrivedPacket);
+
+      [MemberNotNull(nameof(myLastFiredPacket))]
+      protected void requireLastFiredPacket() =>
+         ExceptionAssert.NotNull(myLastFiredPacket);
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs

@@ -1,14 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using System.Collections.Generic;
 
 namespace BCG
 {
    public interface INonIncGroupByEvaluator
    {
-      IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-      string[] get_group_by_cols();
+      IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+      string[]? get_group_by_cols();
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,46 +1,51 @@
-using System;
+using bts;
+using bts.SilverPine;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+using Spark.Core.Disposables;
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
-using System.Text.Json;
-
-using bts;
-using bts.SilverPine;
 
 namespace BCG {
    //[DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+         Func<string[]?> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>?> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
                                           Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
+         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable? rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               var out_rows = _evaluate_func(g_rows);
                if (out_rows != null && out_rows.Count > 0) {
                   rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+                  rv.addRows(out_rows);
                }
-               rv.addRows(out_rows);
             }
             return rv;
          }
-         public string[] get_group_by_cols() {
+         public string[]? get_group_by_cols() {
             return _groupby_cols_func();
          }
 
       }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
+      readonly ViewStream? _main_stream;
+      readonly INonIncGroupByEvaluator? _groupby_func;
+      BlueRowHasher? _row_hasher;
+      readonly string[]? _groupby_cols;
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
+      IDisposable? subscription;
 
       public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -49,7 +54,7 @@ namespace BCG {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
@@ -58,7 +63,7 @@ namespace BCG {
 
       public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
-         Func<string[]> groupby_cols_func = () => {
+         Func<string[]?> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
@@ -75,12 +80,17 @@ namespace BCG {
       }
 
 
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
+      protected override void myOnFirstSubscriber() {
+         ExceptionAssert.IsNull(subscription);
+         ExceptionAssert.NotNull(_main_stream).Subscribe(_main_stream_onchange);
+      }
 
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+      protected override void myOnLastSubscriber() =>
+         Disposable.Dispose(ref subscription);
 
       private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
+         ExceptionAssert.NotNull(_groupby_func);
+         var dp = (IDataPacket)nv;
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -94,7 +104,7 @@ namespace BCG {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         var total_state = _groupby_func.evaluate(grouped_rows);
          _timer.Stop();
          if (total_state is null) {
             Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
@@ -108,8 +118,12 @@ namespace BCG {
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
          // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         if (total_state != null) {
+            var deletedRows = myLastValue is null
+               ? new BlueTable(total_state.schema)
+               : ((IDataPacket)myLastValue).newRows;
+            myFireEvent(new DataPacket(total_state, total_state, deletedRows));
+         }
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs

@@ -1,6 +1,6 @@
 using System;
-using BCG.Utilities;
 using System.Collections;
+using System.Collections.Generic;
 
 namespace BCG
 {
@@ -11,12 +11,10 @@ namespace BCG
    /// </summary>
    public class RowDictionaryTable : IBlueRowCollection
    {
-      protected IDictionary myRowCollections = new Hashtable ();
-      protected BlueTable myLinearCollection;
-      protected string[] myCols;
+      protected readonly Dictionary<string, BlueTable> myRowCollections = new();
+      protected readonly BlueTable myLinearCollection;
+      protected readonly string[] myCols;
       protected IBlueTableSchema mySchema;
-      
-      protected RowDictionaryTable ()  {}
 
       public RowDictionaryTable (IBlueRowCollection rows, params string[] cols)
          : this (rows.schema, cols)
@@ -40,7 +38,7 @@ namespace BCG
 
       public virtual void addRow (IBlueRow row)
       {
-         BlueTable rows = (BlueTable)getGroup (row);
+         var rows = getMutableGroup(row);
          if (rows == null)
          {
             //i.e we dont have any group for this row => we need to create a new group
@@ -53,7 +51,7 @@ namespace BCG
 
       public virtual void removeRow (IBlueRow row)
       {
-         BlueTable rowGroup = (BlueTable)getGroup (row);
+         var rowGroup = requireMutableGroup(row);
 //         BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
          rowGroup.deleteRow (row);
          myLinearCollection.deleteRow (row);
@@ -65,7 +63,7 @@ namespace BCG
 
       public void removeGroup (IBlueRow representativeRow)
       {
-         IBlueRowCollection rowGroup = getGroup (representativeRow);
+         var rowGroup = requireGroup(representativeRow);
          // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
          foreach (IBlueRow row in rowGroup)
             myLinearCollection.deleteRow (row);
@@ -85,24 +83,31 @@ namespace BCG
       /// <returns></returns>
       //public IBlueRowCollection getGroup (IBlueRow row)  {return getGroup (
 
-      public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));}
+      public IBlueRowCollection? getGroup (IBlueRow r) =>
+         getGroup(myRowToString(r));
 
       public ICollection getGroups ()  {return myRowCollections.Values;}
 
-      private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2)
-      {
-         bool retVal = true;
-         foreach (string col in myCols)
-            retVal &= r1[col].Equals(r2[col]);
-         return retVal;
-      }
+      public IBlueRowCollection? getGroup(string repString) =>
+         getMutableGroup(repString);
 
-      public IBlueRowCollection getGroup (string repString)
-      {
-         IBlueRowCollection retVal = myRowCollections[repString] != null ?
-            (IBlueRowCollection) myRowCollections[repString] : null;
-         return retVal;
-      }
+      BlueTable? getMutableGroup(IBlueRow row) =>
+         getMutableGroup(myRowToString(row));
+
+      BlueTable? getMutableGroup(string group) =>
+         myRowCollections.GetValueOrDefault(group);
+
+      IBlueRowCollection requireGroup(IBlueRow row) =>
+         requireGroup(myRowToString(row));
+
+      IBlueRowCollection requireGroup(string group) =>
+         getGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
+
+      BlueTable requireMutableGroup(IBlueRow row) =>
+         requireMutableGroup(myRowToString(row));
+
+      BlueTable requireMutableGroup(string group) =>
+         getMutableGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
 
       private string myRowToString (IBlueRow r)  {return BlueDataUtility.rowToString (r, myCols);}
 

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG
 {
    [DataStreamUserRespresentation (false)]
@@ -17,6 +15,7 @@ namespace BCG
 
       protected override void myInitFiredPacket ()
       {
+         requireSchema();
          myLastFiredPacket = new DataPacket ();
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -25,6 +24,10 @@ namespace BCG
 
       public override void onTableClear (bool isLeft)
       {
+         if (myLastFiredPacket is null) {
+            return;
+         }
+         requireSchema();
          //incase of natural join, left or right wont matter, the end result zeroes out
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs

@@ -1,5 +1,7 @@
+using Spark.Core.Diagnostics;
+
 using System;
-using System.Collections;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BCG
 {
@@ -13,12 +15,10 @@ namespace BCG
       protected JoinOperand myLeftOperand, myRightOperand;
       protected int myLeftJoinColOrdinal, myRightJoinColOrdinal;
       protected ViewStream myLeftStream, myRightStream;
-      protected JoinTableSchema mySchema;
-      protected DataPacket myLastFiredPacket;
+      protected JoinTableSchema? mySchema;
+      protected DataPacket? myLastFiredPacket;
       protected bool myOnceFired = false;
 
-      public IBlueTableSchema schema {get{return mySchema;}}
-
       public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight)
       {
          myLeftStream = vsLeft;
@@ -27,6 +27,9 @@ namespace BCG
          myRightOperand = new JoinOperand (jcRight, false);
       }
 
+      public IBlueTableSchema schema =>
+         requireSchema();
+
       protected override void myOnFirstSubscriber() 
       {
          myLeftOperand.master = this;
@@ -62,29 +65,32 @@ namespace BCG
          {
             myLeftJoinColOrdinal = myLeftOperand.joinColOridinal;
             myRightJoinColOrdinal = myRightOperand.joinColOridinal;
+            myLeftOperand.RequireInitialized();
+            myRightOperand.RequireInitialized();
             mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol, 
                myRightOperand.operandSchema, myRightOperand.joinCol);
             myInitFiredPacket ();
          }
       }
 
+      [MemberNotNull(nameof(mySchema), nameof(myLastFiredPacket))]
       protected virtual void myInitFiredPacket ()
       {
+         var schema = requireSchema();
          myLastFiredPacket = new DataPacket ();
-         myLastFiredPacket.newRows = new JoinTable (mySchema);
-         myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         myLastFiredPacket.deletedRows = new BlueTable (mySchema);
+         myLastFiredPacket.newRows = new JoinTable (schema);
+         myLastFiredPacket.addedRows = new BlueTable (schema);
+         myLastFiredPacket.deletedRows = new BlueTable (schema);
       }
 
-      public IBlueTableSchema getSchema (bool isLeft)
-      {
-         IBlueTableSchema retVal;
-         retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
-         return retVal;
-      }
+      public IBlueTableSchema? getSchema (bool isLeft) =>
+         isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
 
       public void endSession ()
       {
+         if (myLastFiredPacket is null) {
+            return;
+         }
          if (!myOnceFired)
          {
             myOnceFired = true;
@@ -97,13 +103,17 @@ namespace BCG
 
       protected virtual void myInit ()
       {
-         myLeftOperand.init ();
-         myRightOperand.init ();
+         myLeftOperand.Reset ();
+         myRightOperand.Reset ();
          myLastFiredPacket = null;
       }
 
       public virtual void onTableClear (bool isLeft)
       {
+         if (myLastFiredPacket is null) {
+            return;
+         }
+         requireSchema();
          //incase of natural join, left or right wont matter, the end result zeroes out
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinTable (mySchema);
@@ -112,15 +122,19 @@ namespace BCG
 
       public virtual void deleteRows (IBlueRowCollection rows, bool isLeft)
       {
+         var last = myLastFiredPacket;
+         if (last is null) {
+            return;
+         }
          JoinOperand thisOp, otherOp;
          int joinColOrdinal;
          myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft);
          foreach (IBlueRow row in rows)
          {
             thisOp.deleteRow (row);
-            if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0)
+            if (otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)).Count == 0)
                continue;   //row does not participate in join
-            ((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket);
+            ((JoinTable)last.newRows).deleteRow(isLeft, row, last);
          }
       }
 
@@ -135,10 +149,12 @@ namespace BCG
             otherOp = myLeftOperand;
             joinColOrdinal = myRightJoinColOrdinal;
          }
+         var schema = requireSchema();
+         myLastFiredPacket ??= new DataPacket(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema));
          foreach (IBlueRow row in rows)
          {
             thisOp.addRow (row);
-            RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]);
+            RowList matchedNodes = otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal));
             if (matchedNodes.Count == 0)
                continue;   //this row did not participate in the join
             if (isLeft)
@@ -146,10 +162,22 @@ namespace BCG
                   ((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket);
             else
                foreach (RowNode matchedNode in matchedNodes)
-                  ((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket);
+                  ((JoinTable)myLastFiredPacket.newRows).addRow(ExceptionAssert.NotNull(matchedNode.row), row, myLastFiredPacket);
          }
       }
 
+      [MemberNotNull(nameof(mySchema))]
+      protected JoinTableSchema requireSchema() =>
+         ExceptionAssert.NotNull(mySchema);
+
+      [MemberNotNull(nameof(myLastFiredPacket))]
+      protected DataPacket requireLastFiredPacket() {
+         if (myLastFiredPacket is null) {
+            myInitFiredPacket();
+         }
+         return myLastFiredPacket;
+      }
+
       private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft)
       {
          if (isLeft)
@@ -166,6 +194,9 @@ namespace BCG
          }
       }
 
+      object getJoinKey(IBlueRow row, int joinColOrdinal) =>
+         row[joinColOrdinal] ?? throw new ArgumentException("Row has null join key", nameof(row));
+
       /**************************************TEST METHOD(S)*********************************************/
       /// <summary>
       /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -173,8 +204,7 @@ namespace BCG
       /// </summary>
       /// <param name="leftPacket"></param>
       /// <param name="rightPacket"></param>
-      public void test (DataPacket dp, bool isLeft)
-      {
+      public void test(IDataPacket dp, bool isLeft) {
          JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
          jo.changeHandler (dp);
       }

src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs

@@ -1,5 +1,6 @@
-using System;
-using System.Collections;
+using Spark.Core.Diagnostics;
+
+using System.Collections.Generic;
 
 namespace BCG
 {
@@ -8,7 +9,7 @@ namespace BCG
 	/// </summary>
    internal class JoinDeletionIndex
    {
-      IDictionary myIndex = new Hashtable ();
+      Dictionary<object, RowList> myIndex = new();
       private string myOpIdName ;
       private bool myIsLeft;
 
@@ -28,10 +29,8 @@ namespace BCG
       /// <param name="row">JoinRow to add</param>
       public void add (JoinRow row)
       {
-         object o = row[myOpIdName];
-         if (myIndex[o] == null)
-            myIndex[o] = new RowList ();
-         RowNode nodeAdded = ((RowList)myIndex[o]).add (row);
+         object o = ExceptionAssert.NotNull(row[myOpIdName]);
+         RowNode nodeAdded = myIndex.GetOrAdd(o, _ => new()).add(row);
          if (myIsLeft)
             row.leftNode = nodeAdded;
          else
@@ -43,7 +42,10 @@ namespace BCG
       /// To remove a node, if node does not exist exception is thrown
       /// </summary>
       /// <param name="node"></param>
-      public void remove (RowNode node)   {((RowList)myIndex[node.row[myOpIdName]]).remove (node);}
+      public void remove (RowNode node) {
+         var nodeId = ExceptionAssert.NotNull(node.row?[myOpIdName]);
+         myIndex[nodeId].remove(node);
+      }
 
       public RowList getMatchingRows (object key)  {
          RowList retVal = (RowList)myIndex[key];

src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs

@@ -1,19 +1,18 @@
-using System;
-using BCG.Utilities;
-
 using bts;
 
+using Bts.DataStreamCore;
+
 namespace BCG
 {
 	internal class JoinForeignTable : JoinTable
 	{
 		public JoinForeignTable (JoinTableSchema schema) : base (schema) {}
 
-      public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
-      {
-         JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema);
-         Log.assert(myIdIndex[leftRow["id"]] == null);
-         myIdIndex[leftRow["id"]] = newRow;
+      public override void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
+         var newRow = new JoinRow(leftRow, rightRow, mySchema);
+         int leftId = leftRow.Id();
+         Log.assert(myIdIndex[leftId] == null);
+         myIdIndex[leftId] = newRow;
          ((BlueTable)(dp.addedRows)).addRow (newRow);
          myLeftIndex.add (newRow);
          myRightIndex.add (newRow);

src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs

@@ -1,11 +1,12 @@
-using System;
-using System.Collections;
-using BCG.Utilities;
-
 using bts;
 
 using Bts.ColumnMapExpressions;
 
+using Spark.Core.Diagnostics;
+
+using System;
+using System.Collections;
+
 namespace BCG
 {
 	/// <summary>
@@ -37,7 +38,8 @@ namespace BCG
       private JoinTableSchema myJoinSchema;
       private IBlueTableSchema myLeftSchema, myRightSchema;
       private string myLeftColExpression, myRightColExpression;
-      private string[] myUserRules, myResolvedNames;
+      string[] myUserRules;
+      string[]? myResolvedNames;
       private IDictionary myFinalMap = new Hashtable ();
 
 
@@ -87,16 +89,18 @@ namespace BCG
       /// </summary>
       private void myFormUserRules ()
       {
+         var leftExp = ColumnMapExpression.Parse(myLeftColExpression) ?? throw new InvalidOperationException("Failed to parse left column expression");
+         var rightExp = ColumnMapExpression.Parse(myRightColExpression) ?? throw new InvalidOperationException("Failed to parse right column expression");
          if (DataStreamLoadFileConfig.instance.useStaticParser)
          {
-            myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
-            myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
+            myFormUserRules(true, leftExp);
+            myFormUserRules(false, rightExp);
          }
          else
          {
             // TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
-            myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
-            myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
+            myFormUserRules(true, leftExp);
+            myFormUserRules(false, rightExp);
          }
 
       }
@@ -115,13 +119,7 @@ namespace BCG
                OriginalNameInfo oni = myOriginalColInfo[i];
                if (oni.isLeft == isLeft)
                {
-                  if (userMap[oni.colName] != null)
-                  {
-                     if ((string)userMap[oni.colName] != oni.colName)
-                        myUserRules[i] = (string)userMap[oni.colName];
-                     else
-                        myUserRules[i] = oni.colName;
-                  }
+                  myUserRules[i] = userMap[oni.colName] as string ?? oni.colName;
                }
             }
          else
@@ -131,10 +129,13 @@ namespace BCG
                OriginalNameInfo oni = myOriginalColInfo[i];
                if (oni.isLeft == isLeft)
                {
-                  if (userMap[oni.colName] == null)
+                  var colName = userMap[oni.colName] as string;
+                  // myUserRules[i] = userMap[oni.colName] as string 
+                  if (colName is null) {
                      myUserRules[i] = oni.colName;
-                  else if ((string)userMap[oni.colName] != oni.colName)
-                     myUserRules[i] = (string)userMap[oni.colName];
+                  } else if (colName != oni.colName) {
+                     myUserRules[i] = colName;
+                  }
                }
             }
          }
@@ -165,26 +166,12 @@ namespace BCG
 
       private void myFormFinalMap ()
       {
+         ExceptionAssert.NotNull(myResolvedNames);
          foreach (string resolvedName in myResolvedNames)
             if (resolvedName != null)
                myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)];
       }
 
-
-      /// <summary>
-      /// Utility method to invert a hashtable
-      /// </summary>
-      /// <param name="arg"></param>
-      /// <returns></returns>
-   
-      private IDictionary reciprocateDictionary (IDictionary arg)
-      {
-         IDictionary retVal = new Hashtable ();
-         foreach (DictionaryEntry de in arg)
-            retVal[de.Value] = de.Key;
-         return retVal;
-      }
-
       /// <summary>
       /// Utility method to find all occurances of an obkect in an IList
       /// </summary>
@@ -194,9 +181,11 @@ namespace BCG
       private ArrayList myFindAllInCollection (IList list, object obj)
       {
          ArrayList retVal = new ArrayList ();
-         for (int i = 0; i < list.Count; i++)
-            if (list[i] != null && list[i].Equals (obj))
+         for (int i = 0; i < list.Count; i++) {
+            if (Equals(list[i], obj)) {
                retVal.Add (i);
+            }
+         }
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs

@@ -1,6 +1,7 @@
+using Spark.Core.Diagnostics;
+
 using System;
-using System.Collections;
-using BCG.Utilities;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BCG
 {
@@ -11,34 +12,46 @@ namespace BCG
    public class JoinOperand 
    {
       private string myJoinCol;
-      private DataPacket myLastArrivedPacket;
-      private JoinOperand myOtherOperand;
+      IDataPacket? myLastArrivedPacket;
+      JoinOperand? myOtherOperand;
       private bool myIsLeft;
-      private JoinOperandRowIndex myRowIndex;
-      private InternalJoinStream myMaster;
-      private IBlueTableSchema myOpSchema;
-      
+      JoinOperandRowIndex? myRowIndex;
+      InternalJoinStream? myMaster;
+
       public JoinOperand (string joinCol, bool isLeft)
       {
          myJoinCol = joinCol;
          myIsLeft = isLeft;
       }
 
-      internal void changeHandler (object o)
-      {
-         DataPacket temp = (DataPacket)o;
-         if (myLastArrivedPacket != null)
-            mySubsequentChangeHandler ((DataPacket)o);
-         else
-            myFirstChangeHandler ((DataPacket)o);
+      [MemberNotNullWhen(true, nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
+      bool isInitialized {
+         get {
+            if (myLastArrivedPacket is null) {
+               return false;
+            }
+            ExceptionAssert.NotNull(myMaster, myLastArrivedPacket, operandSchema, myOtherOperand, myRowIndex);
+            return true;
+         }
       }
 
-      private void myFirstChangeHandler (DataPacket arrivedDp)
+      internal void changeHandler (object o)
       {
+         var dp = (IDataPacket)o;
+         if (isInitialized) {
+            mySubsequentChangeHandler (dp);
+         } else {
+            myFirstChangeHandler(dp);
+         }
+      }
+
+      [MemberNotNull(nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
+      bool myFirstChangeHandler(IDataPacket arrivedDp) {
+         ExceptionAssert.NotNull(myMaster);
          myLastArrivedPacket = arrivedDp;
-         myOpSchema = myLastArrivedPacket.newRows.schema;
+         operandSchema = myLastArrivedPacket.newRows.schema;
          myOtherOperand = myMaster.getOtherOperand (this);
-         myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo);
+         myRowIndex = new JoinOperandRowIndex (operandSchema[myJoinCol].colNo, operandSchema["id"].colNo);
          if (myOtherOperand.hasReceivedData)
          {
             //the other operand has already received data, so the join would
@@ -49,14 +62,16 @@ namespace BCG
             myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft);
             myMaster.endSession ();
          }
-         else
+         else {
             myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows, 
-               myOpSchema[myJoinCol].colNo,
-               myOpSchema["id"].colNo);
+               operandSchema[myJoinCol].colNo,
+               operandSchema["id"].colNo);
+         }
+         return true;
       }
 
-      private void mySubsequentChangeHandler (DataPacket arrivedDp)
-      {
+      void mySubsequentChangeHandler (IDataPacket arrivedDp) {
+         RequireInitialized();
          myLastArrivedPacket = arrivedDp;
          if (!myOtherOperand.hasReceivedData)
          {
@@ -85,29 +100,46 @@ namespace BCG
          myMaster.endSession ();
       }
 
-      internal void addRow (IBlueRow row) {myRowIndex.add (row);}
+      internal void addRow (IBlueRow row) =>
+         ExceptionAssert.NotNull(myRowIndex).add(row);
 
-      internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);}
+      internal void deleteRow (IBlueRow row) {
+         RequireInitialized();
+         myRowIndex.remove (row);
+      }
 
-      internal IBlueTableSchema operandSchema  {get{return myOpSchema;}}
+      internal IBlueTableSchema? operandSchema { get; private set; }
 
       internal InternalJoinStream master {set{myMaster = value;}}
 
       internal string joinCol   {get{return myJoinCol;}}
 
-      internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}}
+      internal int joinColOridinal {
+         get {
+            RequireInitialized();
+            return operandSchema[myJoinCol].colNo;
+         }
+      }
   
-      internal RowList getMatchingRows (object key)   {return (RowList) myRowIndex.getMatchingRows (key);}
+      internal RowList getMatchingRows (object key) {
+         RequireInitialized();
+         return myRowIndex.getMatchingRows(key);
+      }
 
       internal bool hasReceivedData   {get{return myLastArrivedPacket != null;}}
 
-      internal void init ()  
-      {
+      internal void Reset() {
          ///should be brought back to the the state before the first callback 
          ///(ie the state after construction)
          myRowIndex = null;
          myLastArrivedPacket = null;
       }
-      
+
+      [MemberNotNull(nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
+      internal void RequireInitialized() {
+         if (!isInitialized) {
+            throw new InvalidOperationException("Internal state not properly initialized");
+         }
+      }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs

@@ -1,6 +1,6 @@
-using System;
-using System.Collections;
-using BCG.Utilities;
+using Spark.Core.Diagnostics;
+
+using System.Collections.Generic;
 
 namespace BCG
 {
@@ -11,8 +11,10 @@ namespace BCG
 	/// </summary>
 	internal class JoinOperandRowIndex
 	{
-      private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
-      private int myJoinColOrdinal, myIdColOrdinal;
+      readonly Dictionary<object, RowList> myJoinColDict = new();
+      readonly Dictionary<int, RowNode> myIdDict = new();
+      readonly int myJoinColOrdinal;
+      readonly int myIdColOrdinal;
 
       /// <summary>
       /// 
@@ -34,19 +36,14 @@ namespace BCG
 
       public void add (IBlueRow row)
       {
-         RowList list;
-         if (myJoinColDict[row[myJoinColOrdinal]] == null)
-            myJoinColDict[row[myJoinColOrdinal]] = list = new RowList (); 
-         else
-            list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
-         myIdDict[row[myIdColOrdinal]] = list.add (row);
+         myIdDict[getId(row)] = myJoinColDict.GetOrAdd(getJoinKey(row), _ => new()).add(row);
       }
 
       public void remove (IBlueRow row)
       {
-         RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
-         ((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
-         myIdDict.Remove (row[myIdColOrdinal]);
+         var id = getId(row);
+         myJoinColDict[getJoinKey(row)].remove(myIdDict[id]);
+         myIdDict.Remove(id);
       }
 
       public RowList getMatchingRows (object key)  
@@ -62,5 +59,11 @@ namespace BCG
          myJoinColDict.Clear ();
          myIdDict.Clear ();
       }
+
+      object getJoinKey(IBlueRow row) =>
+         ExceptionAssert.NotNull(row[myJoinColOrdinal]);
+
+      int getId(IBlueRow row) =>
+         (int)row[myIdColOrdinal]!;
 	}
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs

@@ -1,17 +1,21 @@
-using System;
-using BCG.Utilities;
-using System.Collections;
 using bts;
 
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections;
+
 namespace BCG
 {
    internal class JoinRow : IBlueRow 
    {
-      private RowNode myLeftNode, myRightNode;
-      protected IBlueRow myLeftRow, myRightRow;
+      RowNode? myLeftNode;
+      RowNode? myRightNode;
+      protected IBlueRow myLeftRow;
+      protected IBlueRow? myRightRow;
       protected int myId;
       protected JoinTableSchema mySchema;
-      private object[] myValues;
+      private object?[] myValues;
       
       #region Constructor(s)
 
@@ -21,8 +25,8 @@ namespace BCG
       /// <param name="leftRow">left row of the join</param>
       /// <param name="rightRow">right row of the join</param>
       /// <param name="schema">schema of the joined row</param>
-      public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) 
-         : this ((int)leftRow["id"], leftRow, rightRow, schema)   {}
+      public JoinRow (IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) 
+         : this(leftRow.Id(), leftRow, rightRow, schema) { }
 
       /// <summary>
       /// Constructs a row assigning it the ID provioded as the first argument
@@ -31,8 +35,7 @@ namespace BCG
       /// <param name="leftRow">left row of the join</param>
       /// <param name="rightRow">right row of the join</param>
       /// <param name="schema">schema of the joined row</param>
-      public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
-      {
+      public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) {
          myId = id;
          myLeftRow = leftRow;
          myRightRow = rightRow;
@@ -48,13 +51,13 @@ namespace BCG
 
       public bool isPartialJoin{get{return myRightRow == null;}}
 
-      public RowNode leftNode 
+      public RowNode? leftNode 
       {
          get{return myLeftNode;}
          set{myLeftNode = value;}
       }
 
-      public RowNode rightNode
+      public RowNode? rightNode
       {
          get{return myRightNode;}
          set{myRightNode = value;}
@@ -85,7 +88,7 @@ namespace BCG
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public virtual object this [object key]
+      public virtual object? this[object key]
       {
          get
          {
@@ -96,7 +99,7 @@ namespace BCG
          set {throw new NotSupportedException ("The IBlueRow is read only");}
       }
 
-      public object this [int index]
+      public object? this[int index]
       {
          get
          {
@@ -119,7 +122,8 @@ namespace BCG
          }
       }
 
-      public object this [string colName] {get{return this[mySchema[colName].colNo];}}
+      public object? this[string colName] =>
+         this[mySchema[colName].colNo];
    
       public ICollection Keys{get{return mySchema.columnNames;}}
 
@@ -133,7 +137,8 @@ namespace BCG
          }
       }
 
-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add (object key, object? value) =>
+         throw new NotSupportedException ("The IBlueRow is readonly");
 
       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs

@@ -1,20 +1,26 @@
-using System;
-using System.Data;
-using System.Collections;
-using System.Text.RegularExpressions;
-using BCG.Utilities;
-using BCG.Utilities.General;
 using bts;
 
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections;
+
 namespace BCG
 {
    public class JoinStream : ViewStream
    {
-      protected InternalJoinStream? myIjs;
+      protected readonly InternalJoinStream? myIjs;
+      protected readonly ColumnMap? myLeftCm;
+      protected readonly ColumnMap? myRightCm;
+      protected readonly bool myIsProject = true;
       protected ProjectStream? myPs;
-      protected ColumnMap? myLeftCm, myRightCm;
-      protected bool myIsProject = true;
-      private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
+
+      readonly bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
+      IDisposable? internalSubscription;
+      IDisposable? projectSubscription;
       
       protected JoinStream () {}
       public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1,
@@ -111,47 +117,61 @@ namespace BCG
          {
             if (myPs == null)
             {
-               JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression, 
-                  myIjs.getSchema (false), myRightCm.columnExpression);
+               ExceptionAssert.NotNull(myIjs, myLeftCm, myRightCm);
+               var leftSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Left schema is null");
+               var rightSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Right schema is null");
+               var joinSchema = (JoinTableSchema)myIjs.schema;
+               JoinNameResolver resolver = new JoinNameResolver(
+                  joinSchema, leftSchema, myLeftCm.columnExpression, rightSchema, myRightCm.columnExpression);
                IDictionary  map = resolver.getMap ();
                map["id"] = "id";
+               Disposable.Dispose(ref projectSubscription);
                myPs = new ProjectStream (myIjs, map);
-               myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
+               projectSubscription = myPs.Subscribe(myProjectChangeHandler);
                return;
             }
             else
                Log.assert(false, "This call back should not have been called");
          }
-         else
-            myFireEvent ((o as ICloneable).Clone());
+         else {
+            myFireEvent(cloneDataPacket(o));
+         }
       }
 
       private void myProjectChangeHandler (object o)
       {
-         if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
-            myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
-         myFireEvent ((o as ICloneable).Clone());
-         
+         ExceptionAssert.NotNull(myIjs);
+         if (myIjs.numberOfSubscribers > 1) {
+            Disposable.Dispose(ref internalSubscription);
+         }
+         myFireEvent(cloneDataPacket(o));
       }
 
       protected override void myOnFirstSubscriber ()
       {
-         myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         Log.isnull(internalSubscription);
+         internalSubscription = ExceptionAssert.NotNull(myIjs).Subscribe(myInternalJoinChangeHandler);
       }
 
       protected override void myOnLastSubscriber ()
       {
-         // how does this relate to the -= in myProjectChangeHandler?
-         myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
-
+         Disposable.Dispose(ref internalSubscription);
          // if there is a  project after the ijs we need to 
          // unsubscribe to it otherwise references will be kept and 
          // a memory leak will occur
-         if (myPs != null)
-            myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
+         Disposable.Dispose(ref projectSubscription);
          myPs = null;
       }
 
+      object cloneDataPacket(object dpObj) {
+         var rv = (dpObj as ICloneable)?.Clone();
+         if (rv is null) {
+            var dp = (IDataPacket)dpObj;
+            rv = new DataPacket(new BlueTable(dp.newRows), new BlueTable(dp.addedRows), new BlueTable(dp.deletedRows));
+         }
+         return rv;
+      }
+
       /**************************************TEST METHOD(S)*********************************************/
       /// <summary>
       /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -159,7 +179,9 @@ namespace BCG
       /// </summary>
       /// <param name="leftPacket"></param>
       /// <param name="rightPacket"></param>
-      public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);}
+      public void test (DataPacket dp, bool isLeft) {
+         ExceptionAssert.NotNull(myIjs).test(dp, isLeft);
+      }
       /***********************************************************************************************/
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs

@@ -1,14 +1,17 @@
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections;
-using System.IO;
-using BCG.Utilities.General;
+using System.Collections.Generic;
 
 namespace BCG
 {
    internal class JoinTable : IBlueRowCollection
    {
       protected JoinTableSchema mySchema;
-      protected IDictionary myIdIndex = new Hashtable ();
+      protected readonly Dictionary<int, IBlueRow> myIdIndex = new();
       protected JoinDeletionIndex myLeftIndex, myRightIndex;
       private int myNewRowId;
 
@@ -19,19 +22,16 @@ namespace BCG
          myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
       }
 
-      public IBlueRow this [int id] 
-      {
-         get
-         {
-            if (myIdIndex[id] == null)
-               throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
-            return (IBlueRow)myIdIndex[id];
+      public IBlueRow this [int id] {
+         get {
+            var rowObj = myIdIndex[id]
+               ?? throw new BlueDataException($"ID {id} does not exist in table {mySchema.tableName}");
+            return (IBlueRow)rowObj;
          }
       }
 
       #region JoinTable members
-      public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
-      {
+      public virtual void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
          JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
          myIdIndex[myNewRowId ++] = newRow;
          ((BlueTable)(dp.addedRows)).addRow (newRow);
@@ -40,7 +40,7 @@ namespace BCG
             myRightIndex.add (newRow);
       }
 
-      public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, DataPacket dp)
+      public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, IDataPacket dp)
       {
          JoinDeletionIndex thisIndex = myRightIndex, otherIndex = myLeftIndex;
          if (isLeft) 
@@ -48,17 +48,17 @@ namespace BCG
             thisIndex = myLeftIndex;
             otherIndex = myRightIndex;
          }
-         RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]);
+         RowList matchingNodes = thisIndex.getMatchingRows(deletedRow.Id());
          foreach (RowNode node in matchingNodes)
          {
-            JoinRow row = (JoinRow)node.row;
-            RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode;
+            var row = ExceptionAssert.Argument.Is<JoinRow>(node.row);
+            RowNode? correspondingNode = isLeft ? row.rightNode : row.leftNode;
             if (correspondingNode != null)
                otherIndex.remove (correspondingNode); //in case of partial left joinss
-            myIdIndex.Remove (row["id"]);
+            myIdIndex.Remove(row.Id());
             ((BlueTable)dp.deletedRows).addRow (row);
          }
-         thisIndex.removeAll (deletedRow["id"]);
+         thisIndex.removeAll(deletedRow.Id());
       }
       #endregion
       #region ICollection members
@@ -68,7 +68,11 @@ namespace BCG
 
       public object SyncRoot{get{return this;}}
 
-      public void CopyTo (Array array, int index)  {myIdIndex.Values.CopyTo (array, index);}
+      public void CopyTo (Array array, int index) {
+         foreach (var row in myIdIndex.Values) {
+            array.SetValue(row, index++);
+         }
+      }
       #endregion
 
       #region IEnumerable members

src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs

@@ -1,9 +1,8 @@
-using System;
-using System.Collections;
-using BCG.Utilities;
-
 using bts;
 
+using System.Collections.Generic;
+using System.Linq;
+
 namespace BCG
 {
    /// <summary>
@@ -13,10 +12,11 @@ namespace BCG
    public class JoinTableSchema : BlueTableSchema
    {
       private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
-      private IDictionary myLeftNameToAlias = new Hashtable ();
-      private IDictionary myRightNameToAlias = new Hashtable ();
-      private IDictionary myAliasToLeftName = new Hashtable(),
-                           myAliasToRightName = new Hashtable();
+      readonly Dictionary<string, string> myLeftNameToAlias = new();
+      readonly Dictionary<string, string> myRightNameToAlias = new();
+      readonly Dictionary<string, string> myAliasToLeftName = new();
+      readonly Dictionary<string, string> myAliasToRightName = new();
+
       /// <summary>
       /// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
       /// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
@@ -85,9 +85,9 @@ namespace BCG
             renamed = false;
          }
          myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
-         myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"];
-         myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias);
-         myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias);
+         myIndexByColOrder[myIndexByColOrder.Length - 1] = myIndexByColName["id"];
+         myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
+         myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
       }
 
       public string leftJoinCol  {get{return myLeftJc;}}

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,17 +1,21 @@
 using bts;
 using bts.collections;
 
+using Bts.DataStreamCore;
+
 using mbts;
 
+using Spark.Core.Diagnostics;
+using Spark.Core.Disposables;
+
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BCG {
    public class NonIncJoinBookkeeper : IDisposable {
-      public Action on_new_data;
+      public event Action? on_new_data;
       private readonly NonIncJoinSpec _join_spec;
       private readonly int _join_position;
       private readonly int _start_index_in_output;
@@ -20,13 +24,14 @@ namespace BCG {
       private readonly bool _is_tag_along;
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
-      private IBlueTableSchema _schema = null;
+      IBlueTableSchema? _schema;
       private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
       private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
-      private IBlueRow _solitary_row = null;
+      IBlueRow? _solitary_row;
 
-      private int[] _join_col_indices = null,
-                     _data_col_indices = null;
+      int[]? _join_col_indices;
+      int[]? _data_col_indices;
+      IDisposable? subscription;
 
       public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
          _join_spec = the_join_spec;
@@ -38,7 +43,7 @@ namespace BCG {
       }
 
       private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
+      void _fill_missing_data(IDataPacket dp) {
          if (_filled_missing_data)
             return;
          Func<string[], bool> null_or_empty = (str_arr) => {
@@ -52,17 +57,17 @@ namespace BCG {
             return true;
          };
          Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            HashSet<string> schema_col_map = new HashSet<string>(dp.Schema.columnNames);
             List<string> cols_list = new List<string>(col_names);
             cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
             return cols_list.ToArray();
          };
          if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
+            _join_spec.data_col_names = dp.Schema.columnNames;
          }
          _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
          if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
+            _join_spec.data_col_names = dp.Schema.columnNames;
          }
          if (null_or_empty(_join_spec.projected_col_names)) {
             _join_spec.projected_col_names = _join_spec.data_col_names;
@@ -70,20 +75,16 @@ namespace BCG {
       }
 
       public void init() {
-         _join_spec.stream.onChange += _on_stream_cb;
+         ExceptionAssert.MemberIsNull(subscription);
+         subscription = ExceptionAssert.MemberNotNull(_join_spec.stream).Subscribe(_on_stream_cb);
       }
 
       public void _on_stream_cb(object dp_obj) {
          //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
-         DataPacket dp = (DataPacket)dp_obj;
+         var dp = (IDataPacket)dp_obj;
          _fill_missing_data(dp);
-         if (_schema == null) {
-            _schema = dp.schema;
-            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
-            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
-         }
-
+         init(dp);
          if (_is_left_most) {
             _row_list.Clear();
             foreach (IBlueRow r in dp.newRows)
@@ -108,44 +109,68 @@ namespace BCG {
             on_new_data();
       }
 
+      [MemberNotNullWhen(true, nameof(_schema))]
       public bool is_ready { get { return _schema != null; } }
 
       public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+         requireReady();
          for (int i = 0; i < _join_spec.data_col_names.Length; i++)
             cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
       }
 
       public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
 
-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
-         IBlueRow this_row = _row_list[row_index].second;
-         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-      }
+      public void copy_master_row_to_output_row(int row_index, object?[] output_row) =>
+         copyRow(ExceptionAssert.NotNull(_row_list[row_index].second), output_row);
 
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+      public void copy_matching_row_to_output_row(int row_hash, object?[] output_row) {
          if (_is_tag_along) {
             if (_solitary_row != null)
-               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
+               copyRow(_solitary_row, output_row);
             else
                copy_dbnull_to_output_row(output_row);
          } else {
-            IBlueRow this_row = null;
-            if (_row_map.TryGetValue(row_hash, out this_row))
-               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+            if (_row_map.TryGetValue(row_hash, out var this_row))
+               copyRow(this_row, output_row);
             else
                copy_dbnull_to_output_row(output_row);
          }
       }
 
-      private void copy_dbnull_to_output_row(object[] output_row) {
+      void copy_dbnull_to_output_row(object?[] output_row) {
+         requireDataColIndices();
          for (int i = 0; i < _data_col_indices.Length; i++)
             output_row[i + _start_index_in_output] = DBNull.Value;
       }
 
       #region IDisposable
 
-      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
+      public void Dispose() =>
+         Disposable.Dispose(ref subscription);
 
       #endregion
+
+      void copyRow(IBlueRow src, object?[] dst) {
+         requireDataColIndices();
+         BlueDataUtility.copy_to_row(src, _data_col_indices, dst, _start_index_in_output);
+      }
+
+      [MemberNotNull(nameof(_schema), nameof(_join_col_indices), nameof(_data_col_indices))]
+      void init(IDataPacket dp) {
+         _schema ??= dp.Schema;
+         _join_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
+         _data_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
+      }
+
+      [MemberNotNull(nameof(_data_col_indices))]
+      int[] requireDataColIndices() =>
+         ExceptionAssert.NotNull(_data_col_indices);
+
+      [MemberNotNull(nameof(_schema))]
+      void requireReady() {
+         if (!is_ready) {
+            throw new InvalidOperationException("Not ready");
+         }
+      }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -1,22 +1,27 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using mbts;
 
-using mbts;
+using Spark.Core.Diagnostics;
+
+using System;
+using System.Collections.Generic;
 
 namespace BCG {
    // this a collection of data, theres no joining logic in here
    public class NonIncJoinSpec {
-      public ViewStream stream;
+      public ViewStream? stream;
       // join col names are the names on which the streams are joined (for equality),
       // data col names are the names in the original stream
       // projected col names are their new names, ie if its x:l_x, then x is in the 
       // data_col_names and l_x in projected cols
-      public string[] join_col_names, data_col_names, projected_col_names;
+      public string[] join_col_names;
+      public string[] data_col_names;
+      public string[] projected_col_names;
 
-      public NonIncJoinSpec() { }
+      public NonIncJoinSpec() {
+         join_col_names = Array.Empty<string>();
+         data_col_names = Array.Empty<string>();
+         projected_col_names = Array.Empty<string>();
+      }
 
       public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
          : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
@@ -32,7 +37,7 @@ namespace BCG {
          List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
          foreach (Triplet<ViewStream, string, string> r in join_specs) {
             string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = r.third.Split(' ');
+            string[] visible_cols = ExceptionAssert.NotNull(r.third).Split(' ');
             List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
             foreach (string s in visible_cols) {
                string[] toks = s.Split(':');
@@ -44,7 +49,7 @@ namespace BCG {
                   mapped_col_names.Add(toks[1]);
                }
             }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add(new(ExceptionAssert.NotNull(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
          }
          return rv.ToArray();
       }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -5,18 +5,19 @@ using mbts;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BCG {
    [DataStreamUserRespresentation("fastJoin")]
    public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
       private readonly NonIncJoinSpec[] _join_specs;
-      private readonly INonIncJoinInlineMapper _output_mapper;
+      readonly INonIncJoinInlineMapper? _output_mapper;
       private readonly BlueRowHasher _row_hasher;
       private readonly TimeSpan _throttle_period;
       private readonly ViewStreamDatabase _vsd;
 
       private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
-      private IBlueTableSchema _output_schema = null;
+      IBlueTableSchema? _output_schema;
       private DateTime _last_fire_time = DateTime.MinValue;
       private bool _is_dirty = false;
 
@@ -35,10 +36,10 @@ namespace BCG {
                               params Triplet<ViewStream, string, string>[] join_specs)
          : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
 
-      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0)
          : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
 
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0) {
          _join_specs = join_specs;
          _output_mapper = output_mapper;
          _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -88,6 +89,7 @@ namespace BCG {
          return rv;
       }
 
+      [MemberNotNull(nameof(_output_schema))]
       private void _construct_schema() {
          if (_output_schema == null) {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs

@@ -2,11 +2,12 @@ namespace bts {
    using BCG;
    using BCG.Utilities.General;
 
+   using Spark.Core.Diagnostics;
+   using Spark.Core.Disposables;
+
    using System;
    using System.Diagnostics;
 
-   using ZstdSharp.Unsafe;
-
    // functon that joins the 2 rows and returns the outputput row. The function
    // can return a null which means the 2 rows dont need to be joined
    using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
@@ -16,6 +17,10 @@ namespace bts {
       private bool _dirty = false;
       private Stopwatch _stopwatch = new Stopwatch();
       private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
+      IDisposable? subscription;
+      IBlueRowCollection? leftRows;
+      IBlueRowCollection? rightRows;
+
       public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
          _lvs = lvs;
          _rvs = rvs;
@@ -23,46 +28,56 @@ namespace bts {
       }
 
       protected override void myOnFirstSubscriber() {
-         _lvs.onChange += _handle_data;
-         _rvs.onChange += _handle_data;
+         _lvs.onChange += OnLeftChange;
+         _rvs.onChange += OnRightChange;
+         subscription = Disposable.Create(Unsubscribe);
          _vsd.addWorker(this);
+
+         void OnLeftChange(object dp_obj) {
+            leftRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
+            _dirty = true;
+            doWork();
+         }
+
+         void OnRightChange(object dp_obj) {
+            rightRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
+            _dirty = true;
+            doWork();
+         }
+
+         void Unsubscribe() {
+            _lvs.onChange -= OnLeftChange;
+            _rvs.onChange -= OnRightChange;
+         }
       }
 
       protected override void myOnLastSubscriber() {
-         _lvs.onChange -= _handle_data;
-         _rvs.onChange -= _handle_data;
+         subscription?.Dispose();
+         subscription = null;
          _vsd.removeWorker(this);
       }
 
-      private void _handle_data(object dp_obj) {
-         _dirty = true;
-         doWork();
-      }
-
       public void doWork() {
-         Func<bool> all_data_revcd = () => {
-            return _lvs.rows != null && _rvs.rows != null;
-         };
-         _stopwatch.Start();
-         if (_dirty && all_data_revcd()) {
-            _dirty = false;
-            BlueTable rv = null;
-            foreach (IBlueRow lr in _lvs.rows) {
-               foreach (IBlueRow rr in _rvs.rows) {
-                  IBlueRow resr = _join_func(lr, rr);
-                  if (resr == null)
-                     continue;
-                  rv = rv ?? new BlueTable(resr.schema);
-                  rv.addRow(resr);
-               }
-            }
-            _stopwatch.Stop();
-            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
-            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
-            myFireEvent(dp);
+         if (!_dirty || leftRows is null || rightRows is null) {
+            return;
          }
-
+         _stopwatch.Start();
+         _dirty = false;
+         BlueTable? rv = null;
+         foreach (IBlueRow lr in leftRows) {
+            foreach (IBlueRow rr in rightRows) {
+               IBlueRow resr = _join_func(lr, rr);
+               if (resr == null)
+                  continue;
+               rv = rv ?? new BlueTable(resr.schema);
+               rv.addRow(resr);
+            }
+         }
+         _stopwatch.Stop();
+         Log.Gen.info($"ProximityJoinStream left_count: {leftRows.Count} right_count: {rightRows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs");
+         var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+         myFireEvent(dp);
       }
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs

@@ -1,3 +1,5 @@
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections;
 
@@ -9,7 +11,7 @@ namespace BCG
    /// </summary>
    internal class RowList : ICollection
    {
-      private RowNode myFirstPtr;
+      RowNode? myFirstPtr;
       private int myCount = 0;
      
       #region ICollection methods
@@ -67,19 +69,19 @@ namespace BCG
          }
          else if (node.isFirst)
          {
-            myFirstPtr = myFirstPtr.next;
-            myFirstPtr.previous = null;
+            myFirstPtr = ExceptionAssert.NotNull(myFirstPtr).next;
+            ExceptionAssert.NotNull(myFirstPtr).previous = null;
             return;
          }
          else if (node.isLast)
          {
-            node.previous.next = null;
+            ExceptionAssert.NotNull(node.previous).next = null;
             return;
          }
          else if (!node.isFirst && !node.isLast)
          {
-            node.previous.next = node.next;
-            node.next.previous = node.previous;
+            ExceptionAssert.NotNull(node.previous).next = node.next;
+            ExceptionAssert.NotNull(node.next).previous = node.previous;
             return;
          }
          else
@@ -91,14 +93,16 @@ namespace BCG
 
       internal class RowListEnumerator : IEnumerator
       {
-         private RowNode myPointer;
+         RowNode? myPointer;
          private RowList myList;
          private int myPos = -1;
          public RowListEnumerator (RowList list)   {myList = list;}
-         public object Current   {get{return myPointer;}}
+         public object? Current => myPointer;
          public bool MoveNext()
          {
-            myPointer = myPos == -1 ? myList.myFirstPtr : myPointer.next;
+            myPointer = myPos == -1
+               ? myList.myFirstPtr
+               : ExceptionAssert.NotNull(myPointer).next;
             myPos ++;
             return myPointer != null;  
          }
@@ -108,28 +112,24 @@ namespace BCG
 
    internal class RowNode
    {
-      private IBlueRow myRow;
-      private RowNode myNext;
-      private RowNode myPrevious;
-      private RowList myList;
-      
-      public RowNode (IBlueRow row){myRow = row;}
-      public RowNode (IBlueRow row, RowList list) : this (row)  {myList = list;}
-      public bool isLast {get{return myNext == null;}}
-      public bool isFirst {get{return myPrevious == null;}}
-      public RowList list {get{return myList;}}
-      public IBlueRow row
-      {
-         get{return myRow;}
-         set{myRow = value;}
-      }
-      public RowNode next {
-         get{return myNext;}
-         set{myNext = value;}
-      }
-      public RowNode previous  {
-         get{return myPrevious;}
-         set{myPrevious = value;}
-      }
+      public RowNode (IBlueRow row) =>
+         this.row = row;
+
+      public RowNode (IBlueRow row, RowList list)
+         : this(row) =>
+         this.list = list;
+
+      public bool isLast => next is null;
+      public bool isFirst => previous is null;
+      public RowList? list { get; }
+      public IBlueRow? row { get; set; }
+      public RowNode? next { get; set; }
+      public RowNode? previous { get; set; }
+
+      public RowList RequireList() =>
+         ExceptionAssert.MemberNotNull(list);
+
+      public IBlueRow RequireRow() =>
+         ExceptionAssert.MemberNotNull(row);
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs

@@ -1,125 +1,17 @@
+using Bts.DataStreamCore;
+
+using mbts;
+
+using Spark.Core.Collections;
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using mbts;
-using Spark.Core.Collections;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BCG
 {
-   internal class SingleRowJoinDicionary<TKey, TValue> : IDictionary<TKey, TValue>
-   {
-      private TKey myKey;
-      private TValue myValue;
-
-      #region IDictionary<TKey,TValue> Members
-
-      public void Add(TKey key, TValue value)
-      {
-         myKey = key;
-         myValue = value;
-      }
-
-      public bool ContainsKey(TKey key)
-      {
-         return myKey.Equals(key);
-      }
-
-      public ICollection<TKey> Keys
-      {
-         get { return new TKey[] { myKey }; }
-      }
-
-      public bool Remove(TKey key)
-      {
-         bool rv = myKey.Equals(key);
-         myKey = default(TKey);
-         return rv;
-      }
-
-      public bool TryGetValue(TKey key, out TValue value)
-      {
-         bool rv = false;
-         value = (rv = myKey.Equals(key)) ? myValue : default(TValue);
-         return rv;
-      }
-
-      public ICollection<TValue> Values
-      {
-         get { throw new Exception("The method or operation is not implemented."); }
-      }
-
-      public TValue this[TKey key]
-      {
-         get
-         {
-            throw new Exception("The method or operation is not implemented.");
-         }
-         set
-         {
-            throw new Exception("The method or operation is not implemented.");
-         }
-      }
-
-      #endregion
-
-      #region ICollection<KeyValuePair<TKey,TValue>> Members
-
-      public void Add(KeyValuePair<TKey, TValue> item)
-      {
-         throw new Exception("The method or operation is not implemented.");
-      }
-
-      public void Clear()
-      {
-         throw new Exception("The method or operation is not implemented.");
-      }
-
-      public bool Contains(KeyValuePair<TKey, TValue> item)
-      {
-         throw new Exception("The method or operation is not implemented.");
-      }
-
-      public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
-      {
-         throw new Exception("The method or operation is not implemented.");
-      }
-
-      public int Count
-      {
-         get { throw new Exception("The method or operation is not implemented."); }
-      }
-
-      public bool IsReadOnly
-      {
-         get { throw new Exception("The method or operation is not implemented."); }
-      }
-
-      public bool Remove(KeyValuePair<TKey, TValue> item)
-      {
-         throw new Exception("The method or operation is not implemented.");
-      }
-
-      #endregion
-
-      #region IEnumerable<KeyValuePair<TKey,TValue>> Members
-
-      public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
-      {
-         throw new Exception("The method or operation is not implemented.");
-      }
-
-      #endregion
-
-      #region IEnumerable Members
-
-      IEnumerator IEnumerable.GetEnumerator()
-      {
-         throw new Exception("The method or operation is not implemented.");
-      }
-
-      #endregion
-   }
-
    #region Helper class to deal with stream projections
 
    public class JoinProjectHelper
@@ -167,7 +59,7 @@ namespace BCG
       {
          Pair<string, bool> p = myAliasToOriginal[col];
          isLeft = p.second;
-         return p.first;
+         return ExceptionAssert.NotNull(p.first);
       }
    }
 
@@ -178,10 +70,11 @@ namespace BCG
    public class SuperJoinRow : IBlueRow
    {
       private JoinProjectHelper myHelper;
-      private IBlueRow myLeftRow, myRightRow;
+      IBlueRow myLeftRow;
+      IBlueRow? myRightRow;
       int myID;
 
-      public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
+      public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
       {
          myID = id;
          myLeftRow = leftRow;
@@ -189,21 +82,22 @@ namespace BCG
          myHelper = joinProjectHelper;
       }
 
-      public int leftID { get { return (int)myLeftRow["id"]; } }
+      public int leftID => myLeftRow.Id();
 
-      public int rightID { get { return (int)myRightRow["id"]; } }
+      public int rightID => myRightRow.Id();
 
       public IBlueRow leftRow { get { return myLeftRow; } }
 
-      public IBlueRow rightRow { get { return myRightRow; } }
+      public IBlueRow? rightRow => myRightRow;
 
       #region IBlueRow Members
 
       public IBlueTableSchema schema { get { return myHelper.schema; } }
 
-      public object this[int index] { get { return this[schema[index].colName]; } }
+      public object? this[int index] =>
+         this[schema[index].colName];
 
-      public object this[string colName]
+      public object? this[string colName]
       {
          get
          {
@@ -229,7 +123,7 @@ namespace BCG
 
       public bool Contains(object key)
       {
-         return myLeftRow.Contains(key) || myRightRow.Contains(key);
+         return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
       }
 
       public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
@@ -242,11 +136,11 @@ namespace BCG
 
       public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); }
 
-      public System.Collections.ICollection Values
+      public ICollection Values
       {
          get
          {
-            List<object> rv = new List<object>();
+            var rv = new List<object?>(Keys.Count);
             foreach (string col in Keys)
                rv.Add(this[col]);
             return rv;
@@ -291,9 +185,10 @@ namespace BCG
                                           myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
       private bool myManageUnmappedLeftRows = false;
       protected string myLeftProj = String.Empty, myRightProj = String.Empty;
-      protected IBlueTableSchema myLeftSchema, myRightSchema;
-      protected JoinProjectHelper myJoinProjectHelper;
-      protected BlueTable myState;
+      protected IBlueTableSchema? myLeftSchema;
+      protected IBlueTableSchema? myRightSchema;
+      protected JoinProjectHelper? myJoinProjectHelper;
+      protected BlueTable? myState;
       protected int myID;
       protected ViewStream myLeftStream, myRightStream;
       protected string myLeftKeyCol, myRightKeyCol;
@@ -346,23 +241,19 @@ namespace BCG
 
       private void myHandleLeftStreamChanges(object newValue)
       {
-         myHandleStreamChanges(newValue as DataPacket, true);
+         myHandleStreamChanges((IDataPacket)newValue, true);
       }
 
       private void myHandleRightStreamChanges(object newValue)
       {
-         myHandleStreamChanges(newValue as DataPacket, false);
+         myHandleStreamChanges((IDataPacket)newValue, false);
       }
 
-      private void myHandleStreamChanges(DataPacket dp, bool isLeft)
+      private void myHandleStreamChanges(IDataPacket dp, bool isLeft)
       {
-         myInitSchema(dp.schema, isLeft);
-         BlueTable dels = null, adds = null;
-         if (myState != null)
-         {
-            dels = new BlueTable(myState.schema);
-            adds = new BlueTable(myState.schema);
-         }
+         myInitSchema(dp.Schema, isLeft);
+         var dels = new BlueTable(myState.schema);
+         var adds = new BlueTable(myState.schema);
          myDeleteRows(dp.deletedRows, isLeft, myState, dels);
          myAddRows(dp.addedRows, isLeft, myState, adds);
          if (myState != null)
@@ -373,6 +264,7 @@ namespace BCG
          }
       }
 
+      [MemberNotNull(nameof(myState))]
       private void myInitSchema(IBlueTableSchema schema, bool isLeft)
       {
          if (myState == null)
@@ -398,10 +290,12 @@ namespace BCG
                }
             }
          }
+         ExceptionAssert.NotNull(myState);
       }
 
       private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
       {
+         ExceptionAssert.NotNull(myJoinProjectHelper);
          object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
             otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
          Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
@@ -410,8 +304,8 @@ namespace BCG
                                                          otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
          foreach (IBlueRow r in addedRows)
          {
-            object currentKey = r[thisKeyCol];
-            int currentID = (int)r["id"];
+            object currentKey = ExceptionAssert.NotNull(r[thisKeyCol]);
+            int currentID = r.Id();
             ///add it to 1.
             if (!thisKeyToRows.ContainsKey(currentKey))
                thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
@@ -422,14 +316,14 @@ namespace BCG
                ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
                foreach (IBlueRow mr in matchingRows)
                {
-                  int mrID = (int)mr["id"];
+                  int mrID = mr.Id();
                   SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
                   state.addRow(rr);
                   addedRowsOut.addRow(rr);
                   ///add to 3.
                   if (!thisIDToResultRows.ContainsKey(currentID))
                      thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
-                  thisIDToResultRows[currentID].Add((int)rr["id"], rr);
+                  thisIDToResultRows[currentID].Add(rr.Id(), rr);
                   ///add to 4.
                   if (!otherIDToResultRows.ContainsKey(mrID))
                   {
@@ -437,7 +331,7 @@ namespace BCG
                      if (myManageUnmappedLeftRows && !isLeft)
                         myTakeCareOfUnmatachedLeftRows(mr, false);
                   }
-                  otherIDToResultRows[mrID].Add((int)rr["id"], rr);
+                  otherIDToResultRows[mrID].Add(rr.Id(), rr);
                }
             }
             else
@@ -458,22 +352,22 @@ namespace BCG
                                                          otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
          foreach (IBlueRow r in deletedRows)
          {
-            int id = (int)r["id"];
-            object key = r[thisKeyCol];
+            int id = r.Id();
+            object key = ExceptionAssert.NotNull(r[thisKeyCol]);
             ///delete it from 1.
             thisKeyToRows[key].Remove(id);
             if (thisKeyToRows[key].Count == 0)
                thisKeyToRows.Remove(key);
             ///form return rows and delete from 3.
-            if (thisIDToResultRows.ContainsKey(id))
+            if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRowMap))
             {
-               ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
+               var removedResultRows = (ICollection)removedResultRowMap.Values;
                thisIDToResultRows.Remove(id);
                foreach (SuperJoinRow rr in removedResultRows)
                {
                   ///delete from 4.
                   int otherId = isLeft ? rr.rightID : rr.leftID;
-                  otherIDToResultRows[otherId].Remove((int)rr["id"]);
+                  otherIDToResultRows[otherId].Remove(rr.Id());
                   if (otherIDToResultRows[otherId].Count == 0)
                      otherIDToResultRows.Remove(otherId);
                   delRowsOut.addRow(rr);
@@ -493,7 +387,7 @@ namespace BCG
 
       private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
       {
-         int lid = (int)leftRow["id"];
+         int lid = leftRow.Id();
          if (add)
             myLeftUnmatchedRowsAdds.Add(lid, leftRow);
          else
@@ -501,7 +395,7 @@ namespace BCG
             if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
                myLeftUnmatchedRowsAdds.Remove(lid);
             else
-               myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
+               myLeftUnmatchedRowsDels.Add(lid, leftRow);
          }
       }
    }
@@ -522,44 +416,49 @@ namespace BCG
 
       protected override void myFireEvent(object o)
       {
-         myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket);
+         myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, (IDataPacket)o);
       }
 
-      private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
-                                       Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
-      {
-         BlueTable dels = resultPacket.deletedRows as BlueTable,
-                     adds = resultPacket.addedRows as BlueTable;
+      void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels, Dictionary<int, IBlueRow> unmatchedLeftRowAdds, IDataPacket resultPacket) {
+         ExceptionAssert.NotNull(myJoinProjectHelper);
+         var dels = resultPacket.deletedRows;
+         var adds = resultPacket.addedRows;
 
-         foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
-         {
-            ///an lr will be here if its actually deleted or
-            ///a right row is added, so now it has a partner
-            ///and is no longer unmatched.
-            ///in both cases the half row that this lr
-            ///created needs to be removed from this
-            ///class's state and added to dels
-            ///it means a right is added
-            int lid = (int)lr["id"];
-            SuperJoinRow rr = myLeftIDToHalfRows[lid];
-            myLeftIDToHalfRows.Remove(lid);
-            myState.deleteRow(rr);
-            dels.addRow(rr);
+         if (unmatchedLeftRowsDels.Count > 0) {
+            ExceptionAssert.NotNull(myState);
+            var mutableDels = (BlueTable)dels;
+            foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
+               ///an lr will be here if its actually deleted or
+               ///a right row is added, so now it has a partner
+               ///and is no longer unmatched.
+               ///in both cases the half row that this lr
+               ///created needs to be removed from this
+               ///class's state and added to dels
+               ///it means a right is added
+               int lid = lr.Id();
+               SuperJoinRow rr = myLeftIDToHalfRows[lid];
+               myLeftIDToHalfRows.Remove(lid);
+               myState.deleteRow(rr);
+               mutableDels.addRow(rr);
+            }
          }
 
-         foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
-         {
-            ///an lr will be here if its added with no 
-            ///matching right, or a right matching row
-            ///is gone so it is unmatched now.
-            ///in both cases a half row needs to be
-            ///added to this class's state and added 
-            ///to adds
-            int lid = (int)lr["id"];
-            SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
-            myLeftIDToHalfRows.Add(lid, rr);
-            myState.addRow(rr);
-            adds.addRow(rr);
+         if (unmatchedLeftRowAdds.Count > 0) {
+            ExceptionAssert.NotNull(myState);
+            var mutableAdds = (BlueTable)adds;
+            foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
+               ///an lr will be here if its added with no 
+               ///matching right, or a right matching row
+               ///is gone so it is unmatched now.
+               ///in both cases a half row needs to be
+               ///added to this class's state and added 
+               ///to adds
+               int lid = lr.Id();
+               SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
+               myLeftIDToHalfRows.Add(lid, rr);
+               myState.addRow(rr);
+               mutableAdds.addRow(rr);
+            }
          }
          base.myFireEvent(new DataPacket(myState, adds, dels));
       }

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -1,6 +1,4 @@
 namespace BCG {
-    using System.Collections.Generic;
-
     public class KeyValueToRelationalStream : ViewStream {
         private readonly ViewStream _kv_str;
         private readonly string _doc_id_col, _key_col, _val_col;
@@ -20,7 +18,7 @@ namespace BCG {
         }
 
         private void _on_str_update(object dp_obj) {
-            DataPacket dp = dp_obj as DataPacket;
+            var dp = (IDataPacket)dp_obj;
             var rows = dp.newRows;
             var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
             var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
@@ -28,4 +26,4 @@ namespace BCG {
         }
 
     }
-}
+}

src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinForeignStream.cs

@@ -1,3 +1,5 @@
+using Spark.Core.Diagnostics;
+
 using System;
 
 namespace BCG
@@ -18,16 +20,21 @@ namespace BCG
 
       public override void onTableClear (bool isLeft)
       {
+         requireSchema();
+         requireLastFiredPacket();
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         if (!isLeft)
+         if (!isLeft) {
+            ExceptionAssert.NotNull(myLeftDataPacket);
             foreach (IBlueRow row in myLeftDataPacket.newRows)
                ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
+         }
       }
 
       protected override void myInitFiredPacket ()
       {
+         requireSchema();
          myLastFiredPacket = new DataPacket ();
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);

src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs

@@ -1,6 +1,9 @@
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+
 using System;
-using System.Collections;
-using BCG.Utilities;
+using System.Collections.Generic;
 
 namespace BCG
 {
@@ -11,9 +14,11 @@ namespace BCG
    [DataStreamUserRespresentation (false)]
    public class LeftInternalJoinStream : InternalJoinStream
    {
-      protected DataPacket myLeftDataPacket, myRightDataPacket;
+      protected IDataPacket? myLeftDataPacket;
+      protected IDataPacket? myRightDataPacket;
       //Container to contain all the left rows that are orphaned during a deletion operation
-      private IDictionary myLeftOrphanedRows;
+      Dictionary<int, RowNode>? myLeftOrphanedRows;
+
       public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) :
          base (vsLeft, jcLeft, vsRight, jcRight)   {}
 
@@ -62,7 +67,9 @@ namespace BCG
       private void myTestAndFire (bool isLeft)
       {
          JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
-         DataPacket dp = isLeft ? myLeftDataPacket : myRightDataPacket;
+         var dp = isLeft
+            ? ExceptionAssert.NotNull(myLeftDataPacket)
+            : ExceptionAssert.NotNull(myRightDataPacket);
          //to test whether both have received call backs
          if (myLeftOperand.hasReceivedData)
             jo.changeHandler (dp);
@@ -80,11 +87,13 @@ namespace BCG
       /// <param name="isLeft"></param>
       public override void addRows (IBlueRowCollection rows, bool isLeft)
       {
+         ExceptionAssert.NotNull(myLastFiredPacket);
          foreach (IBlueRow row in rows)
             myAddRow (row, isLeft);
          if (!isLeft)
-            foreach (RowNode node in myLeftOrphanedRows.Values)
-               ((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket);
+            foreach (RowNode node in ExceptionAssert.NotNull(myLeftOrphanedRows).Values) {
+               ((JoinTable)myLastFiredPacket.newRows).addRow(node.RequireRow(), null, myLastFiredPacket);
+            }
       }
 
       /// <summary>
@@ -94,17 +103,19 @@ namespace BCG
       /// <param name="isLeft"></param>
       public override void deleteRows (IBlueRowCollection rows, bool isLeft)
       {
-         myLeftOrphanedRows = new Hashtable ();
+         myLeftOrphanedRows = new();
          foreach (IBlueRow row in rows)
             myDeleteRow (row, isLeft);
       }
 
       private void myAddRow (IBlueRow row, bool isLeft)
       {
+         ExceptionAssert.NotNull(myLastFiredPacket);
          if (isLeft)
          {
             myLeftOperand.addRow (row);
-            RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]);
+            var joinKey = ExceptionAssert.NotNull(row[myLeftJoinColOrdinal]);
+            RowList matchedRows = myRightOperand.getMatchingRows(joinKey);
             if (matchedRows.Count == 0)
                ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
             else 
@@ -113,22 +124,25 @@ namespace BCG
          }
          else
          {
-            RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
-            int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
+            var joinKey = ExceptionAssert.NotNull(row[myRightJoinColOrdinal]);
+            RowList matchedRows = myLeftOperand.getMatchingRows(joinKey);
+            int rightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count;
             myRightOperand.addRow (row);
-            int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
+            int newRightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count;
             if (matchedRows.Count != 0){
                foreach (RowNode matchedNode in matchedRows)
                {
+                  ExceptionAssert.NotNull(matchedNode.row);
                   //if the left rows were orphaned in this transaction (ie in this callback), they should
                   //just be removed from the orphanedlist and we dont need to worry about the consistency of
                   //the jointable as a new row will replace it
                   if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0)
                   {
-                     if (myLeftOrphanedRows[matchedNode.row["id"]] == null)
-                        ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket);
-                     else
-                        myLeftOrphanedRows.Remove (matchedNode.row["id"]);
+                     ExceptionAssert.NotNull(myLeftOrphanedRows);
+                     int matchedId = matchedNode.row.Id();
+                     if (!myLeftOrphanedRows.Remove(matchedId)) {
+                        ((JoinTable)myLastFiredPacket.newRows).deleteRow(true, matchedNode.row, myLastFiredPacket);
+                     }
                   }
                   //they were orphaned by a previous transaction and hence the orphans need to be
                   //removed from the Jointable and the complete row added
@@ -141,41 +155,52 @@ namespace BCG
 
       private void myDeleteRow (IBlueRow row, bool isLeft)
       {
+         var last = requireLastFiredPacket();
          if (isLeft)
          {
             myLeftOperand.deleteRow (row);
-            ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, row, myLastFiredPacket);
+            ((JoinTable)last.newRows).deleteRow (true, row, myLastFiredPacket);
          }
          else
          {
-            RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
-            int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
+            var rightJoinKey = row[myRightJoinColOrdinal] ?? throw new InvalidOperationException("RightJoinKey is null");
+            RowList matchedRows = myLeftOperand.getMatchingRows(rightJoinKey);
+            int rightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count;
             myRightOperand.deleteRow (row);
-            int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
+            int newRightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count;
             if (matchedRows.Count != 0)
             {
-               if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0)
+               if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) {
+                  ExceptionAssert.MemberNotNull(myLeftOrphanedRows);
                //ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to
                //be handled specially, ie joined with nulls, if there are no right rows with this
                //joincolumn value being inserted. 
+                  ExceptionAssert.NotNull(myLeftOrphanedRows);
                   foreach (RowNode matchedNode in matchedRows)
                   {
                      // BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null);
-                     myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode;
+                     myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode;
                   }
-               ((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket);
+               }
+               ((JoinTable)last.newRows).deleteRow (false, row, myLastFiredPacket);
             }
          }           
       }
       
       public override void onTableClear (bool isLeft)
       {
+         if (myLastFiredPacket is null) {
+            return;
+         }
+         requireSchema();
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         if (!isLeft)
+         if (!isLeft) {
+            ExceptionAssert.MemberNotNull(myLeftDataPacket);
             foreach (IBlueRow row in myLeftDataPacket.newRows)
                ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
+         }
       }
       
    }

src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs

@@ -30,7 +30,7 @@ namespace BCG
       /// <summary>
       /// Node containing the referencedRow
       /// </summary>
-      IBlueRow referencedRow {get;}
+      IBlueRow? referencedRow { get; }
 
       /// <summary>
       /// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row