Cleaning market data streams

In other words, don't fire if addedRows and deletedRows are both empty.

.config/dotnet-tools.json

@@ -25,6 +25,12 @@
       "commands": [
         "fake"
       ]
+    },
+    "dotnet-t4": {
+      "version": "3.0.0",
+      "commands": [
+        "t4"
+      ]
     }
   }
 }

.devcontainer/devcontainer.json

@@ -1,5 +1,11 @@
 {
-  "dockerFile": "Dockerfile",
+  "build": {
+    "dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
+    "options": [
+      "--secret",
+      "id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
+    ]
+  },
   "hostRequirements": {
     "memory": "8gb",
     "storage": "10gb"
@@ -22,6 +28,7 @@
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
+        "ms-python.python",
         "ms-vscode.cmake-tools",
         "ms-vscode.cpptools",
         "xaver.clang-format"

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.dotnet-cli/.nuget/packages/microsoft.net.test.sdk/17.10.0/build/netcoreapp3.1/Microsoft.NET.Test.Sdk.Program.cs

@@ -0,0 +1,4 @@
+// <auto-generated> This file has been auto generated. </auto-generated>
+using System;
+[Microsoft.VisualStudio.TestPlatform.TestSDKAutoGeneratedCode]
+class AutoGeneratedProgram {static void Main(string[] args){}}

.editorconfig

@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles
 
-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case
 
-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion
 
-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion
 
-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion

.env

@@ -46,4 +46,11 @@ SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
 
+### PubSubServer ###
+PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
+PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+
+### MdPublisher ###
+MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
+MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -55,6 +55,8 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
+/src/**/*appsettings.Local.json
+/src/**/*/Configurations/Local/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/
@@ -62,3 +64,9 @@ trace_data/
 *.lst
 .env.local
 .secrets
+.dotnet-cli/
+.dotnet-cli/
+.dotnet-cli /
+.dotnet/
+.codex_restore*.log
+src/cs/Bts.Spark.Core/.codexobj/

.vscode/launch.json

@@ -14,6 +14,49 @@
       "type": "coreclr",
       "request": "attach"
     },
+    {
+      "name": "Api Server",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "preLaunchTask": "Build Api Server (Debug)",
+      "windows": {
+        "program": "spark_api_server.exe",
+      },
+      "linux": {
+        "program": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Api Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "spark_api_server.exe",
+      },
+      "linux": {
+        "program": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -26,11 +69,28 @@
       "windows": {
         "program": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DataStreamPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -82,17 +142,30 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "preLaunchTask": "Build DataStream (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "DataStreamQueryAnalyzer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
         },
       },
       "stopAtEntry": false,
@@ -118,6 +191,123 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "PubSubServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      // "env": {
+      //   "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      // },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DSPublisher",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "DSPublisher",
+      },
+      "windows": {
+        "program": "DSPublisher.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DSPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DSPublisher",
+      },
+      "windows": {
+        "program": "DSPublisher.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PersistenceServer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "PersistenceServer",
+      },
+      "windows": {
+        "program": "PersistenceServer.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PersistenceServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PersistenceServer",
+      },
+      "windows": {
+        "program": "PersistenceServer.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sdk Demo",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "preLaunchTask": "Build Api Server (Debug)",
+      "windows": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "linux": {
+        "program": "Bts.Spark.Sdk.Demo",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sdk Demo (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "linux": {
+        "program": "Bts.Spark.Sdk.Demo.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
     {
       "name": "Spark",
       "type": "coreclr",
@@ -127,6 +317,9 @@
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -139,6 +332,9 @@
       "windows": {
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -200,6 +396,7 @@
       "program": "",
       "linux": {
         "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "preLaunchTask": "Build Sparkd (Debug | Clang)",
       },
       "args": [
         "run"
@@ -249,8 +446,38 @@
         }
       ]
     },
+    {
+      "name": "Sparkd Attach",
+      "type": "cppdbg",
+      "request": "attach",
+      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
   ],
   "inputs": [
+    {
+      "id": "apiServerEnvironment",
+      "type": "pickString",
+      "description": "Api Server Configuration",
+      "options": [
+        "Development",
+        "Local",
+        "Staging",
+      ],
+      "default": "Development",
+    },
     {
       "id": "dataStreamPublisherConfig",
       "type": "pickString",
@@ -263,6 +490,26 @@
       ],
       "default": "MarketDataPublisher",
     },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "dataStreamPublisherEnvironment",
+      "type": "pickString",
+      "description": "DataStreamPublisher Configuration",
+      "options": [
+        "Local",
+        "ReplayMarketData",
+      ],
+      "default": "Local",
+    },
     {
       "id": "pubSubServerConfig",
       "type": "pickString",
@@ -273,12 +520,33 @@
       ],
       "default": "PubSubServer",
     },
+    {
+      "id": "sdkDemoEnvironment",
+      "type": "pickString",
+      "description": "Sdk Demo Configuration",
+      "options": [
+        "Development",
+        "Local",
+        "Staging",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkConfigServer",
       "type": "promptString",
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
+    {
+      "id": "sparkEnvironment",
+      "type": "pickString",
+      "description": "Spark Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkTest",
       "type": "promptString",

.vscode/python.env

@@ -1 +1 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py

.vscode/tasks.json

@@ -172,6 +172,66 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Build Spark Sdk",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark Sdk (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark Sdk (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark_sdk.cmd",
+      },
+      "linux": {
+        "command": "tools/build_spark_sdk.sh",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Build Sparkd",
       "group": "build",
@@ -351,6 +411,129 @@
         "id": "wrench"
       },
     },
+    {
+      "label": "Run Api Server",
+      "dependsOn": "Build Api Server",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Debug)",
+      "dependsOn": "Build Api Server (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Release)",
+      "dependsOn": "Build Api Server (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run DataStreamQueryAnalyzer",
       "dependsOn": "Build DataStream",
@@ -358,12 +541,11 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -378,12 +560,11 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -398,12 +579,11 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
         },
       },
       "icon": {
@@ -422,12 +602,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -445,12 +624,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -468,12 +646,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -491,12 +668,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -514,12 +690,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -536,12 +711,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -559,12 +733,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -581,12 +754,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -852,6 +1024,17 @@
     },
   ],
   "inputs": [
+    {
+      "id": "apiServerEnvironment",
+      "type": "pickString",
+      "description": "Api Server Configuration",
+      "options": [
+        "Development",
+        "Local",
+        "Staging",
+      ],
+      "default": "Development",
+    },
     {
       "id": "buildConfiguration",
       "type": "pickString",
@@ -862,6 +1045,16 @@
       ],
       "default": "Debug",
     },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkdCmakePreset",
       "type": "pickString",
@@ -885,26 +1078,30 @@
       "default": "sparkd-debug-clang",
     },
     {
-      "id": "dataStreamPublisherConfig",
+      "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
+        "Development",
+        "MarketData",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sparkConfigQuery",

Bts.Spark.Sdk.sln

@@ -0,0 +1,102 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.ActiveCfg = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.Build.0 = Debug|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.Build.0 = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.Build.0 = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.ActiveCfg = Release|x64
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.ActiveCfg = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.Build.0 = Debug|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.Build.0 = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.ActiveCfg = Release|x64
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.Build.0 = Release|x64
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.Build.0 = Debug|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.Build.0 = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.ActiveCfg = Release|Any CPU
+		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.Build.0 = Debug|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.Build.0 = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.ActiveCfg = Release|Any CPU
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.Build.0 = Debug|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+	EndGlobalSection
+EndGlobal

CMakeLists.txt

@@ -21,7 +21,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -40,9 +40,8 @@ include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include
 link_directories(${EXTERNALS_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
-if(DEFINED APP_DATA_DIR_OVERRIDE)
-  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
+if(DEFINED APP_DATA_DIR)
+  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)

CMakePresets.json

@@ -8,36 +8,41 @@
 
   "configurePresets": [
     {
-      "name": "spark",
+      "name": "spark-base",
+      "hidden": true,
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    },
+
+    {
+      "name": "spark-windows",
+      "inherits": "spark-base",
+      "displayName": "Spark",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "spark"
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
       }
     },
 
@@ -53,8 +58,6 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
-        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "OFF"
@@ -77,8 +80,7 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "sparkd"
+        }
       }
     },
 
@@ -157,22 +159,17 @@
     },
 
     {
-      "name": "spark-api-server-windows",
-      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)",
-      "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
-        "SHARED_MEMORY_PREFIX": "spark_api_server"
-      }
-    },
-
-    {
-      "name": "spark-api-server-linux",
-      "inherits": "sparkd",
-      "displayName": "Spark Api Server (Linux)",
+      "name": "spark-linux",
+      "inherits": "spark-base",
+      "displayName": "Spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "platform": "linux",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -181,21 +178,26 @@
           "type": "BOOL",
           "value": "OFF"
         },
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
         "CMAKE_EXPORT_COMPILE_COMMANDS": {
           "type": "BOOL",
           "value": "ON"
         },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "EXPORT_MD_CONSUMER": {
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
+        "DATABENTO_USE_EXTERNAL_JSON": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "EXTERNALS_ROOT": "${sourceDir}/externals/linux"
       }
     },
 
@@ -215,92 +217,32 @@
       "cacheVariables": {
         "CCAPI_LOG_LEVEL": "Trace"
       }
-    },
-
-    {
-      "name": "data-stream-suite-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
-      "inherits": "spark",
-      "displayName": "Data Stream Suite (Windows)",
-      "description": "Data Stream utils ported over from Edge",
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "SHARED_MEMORY_PREFIX": "data_stream"
-      }
-    },
-
-    {
-      "name": "data-stream-suite-linux",
-      "displayName": "Data Stream Suite (Linux)",
-      "description": "Data Stream utils ported over from Edge",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
-      "inherits": "sparkd",
-      "generator": "Ninja Multi-Config",
-      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "BUILD_ADMIN_WEB_APP": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SDK": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SPARKD": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_C_COMPILER": "clang",
-        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "data_stream"
-      }
     }
   ],
 
   "buildPresets": [
     {
-      "name": "spark",
+      "name": "spark-windows",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
-      "hidden": true
+      "hidden": true,
+      "configurePreset": "spark-windows"
     },
 
     {
-      "name": "spark-debug",
+      "name": "spark-windows-debug",
       "displayName": "Spark (Debug)",
-      "inherits": "spark",
-      "configurePreset": "spark",
+      "inherits": "spark-windows",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-release",
+      "name": "spark-windows-release",
       "displayName": "Spark (Release)",
-      "inherits": "spark",
-      "configurePreset": "spark",
+      "inherits": "spark-windows",
       "configuration": "Release"
     },
 
@@ -343,52 +285,27 @@
     },
 
     {
-      "name": "spark-api-server-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
-      "hidden": true,
-      "configurePreset": "spark-api-server-windows"
-    },
-
-    {
-      "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Windows | Debug)",
-      "inherits": "spark-api-server-windows",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Windows | Release)",
-      "inherits": "spark-api-server-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "spark-api-server-linux",
+      "name": "spark-linux",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
       "hidden": true,
-      "configurePreset": "spark-api-server-linux"
+      "configurePreset": "spark-linux"
     },
 
     {
-      "name": "spark-api-server-linux-debug",
-      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Debug)",
+      "name": "spark-linux-debug",
+      "inherits": "spark-linux",
+      "displayName": "Spark (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-api-server-linux-release",
-      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Release)",
+      "name": "spark-linux-release",
+      "inherits": "spark-linux",
+      "displayName": "Spark (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -405,34 +322,6 @@
       "configurePreset": "sparkd-ccapi-diag",
       "displayName": "Debug (ccapi diag)",
       "inherits": "sparkd"
-    },
-
-    {
-      "name": "data-stream-suite-windows-debug",
-      "displayName": "Data Stream Suite (Windows | Debug)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-windows-release",
-      "displayName": "Data Stream Suite (Windows | Release)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "data-stream-suite-linux-debug",
-      "displayName": "Data Stream Suite (Linux | Debug)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-linux-release",
-      "displayName": "Data Stream Suite (Linux | Release)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Release"
     }
   ]
 }

Spark.Api.Server.sln

@@ -1,10 +1,11 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
@@ -42,168 +43,538 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
+		MinSizeRel|Any CPU = MinSizeRel|Any CPU
+		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
+		Release|Any CPU = Release|Any CPU
+		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
+		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
+		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.Build.0 = Debug|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.ActiveCfg = Release|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.Build.0 = Release|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.Build.0 = Debug|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.ActiveCfg = Release|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.Build.0 = Release|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.Build.0 = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.Build.0 = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.Build.0 = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.Build.0 = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.Build.0 = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.Build.0 = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.Build.0 = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.Build.0 = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.Build.0 = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.Build.0 = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.Build.0 = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.Build.0 = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.Build.0 = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.Build.0 = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.Build.0 = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.Build.0 = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.Build.0 = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.Build.0 = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.Build.0 = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.Build.0 = Release|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.Build.0 = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.Build.0 = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.ActiveCfg = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.Build.0 = Release|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
@@ -215,8 +586,17 @@ Global
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
 	EndGlobalSection
 EndGlobal

cmake/Protobuf.cmake

@@ -1,164 +0,0 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
-set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
-
-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    add_custom_command(
-      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
-      DEPENDS ${ABS_FIL}
-      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM)
-  endforeach()
-
-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
-      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
-      PARENT_SCOPE)
-endfunction()
-
-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
-
-    set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
-    )
-
-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
-      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
-        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
-        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
-    else()
-      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
-        ARGS
-          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
-    endif()
-
-  endforeach()
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
-      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
-      PARENT_SCOPE)
-
-endfunction()
-

compose.yaml

@@ -1,7 +1,7 @@
 services:
   sparkd:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -12,19 +12,19 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
+        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
+          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+        if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
           set -a
-          . /home/bts/sparkdata/conf/.env.local
+          . /home/bts/.local/share/bts/sparkd/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
+          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
         fi
         ./sparkd run"
     env_file:
@@ -41,8 +41,8 @@ services:
 
   sparkd-symbols:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -53,21 +53,21 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
+        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
+          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        ls -alh /home/bts/sparkdata/conf/
-        cat /home/bts/sparkdata/conf/.env.local
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+        ls -alh /home/bts/.local/share/bts/sparkd/conf/
+        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
+        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
           set -a
-          . /home/bts/sparkdata/conf/.env.local
+          . /home/bts/.local/share/bts/sparkd/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
+          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
         fi
         ./sparkd run"
     env_file:
@@ -86,15 +86,15 @@ services:
           memory: "${SPARKD__MEMORY:-4gb}"
 
   sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
+    image: bluetradingsystems/linux_dotnet8
     volumes:
       - type: volume
         read_only: true
         source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
+        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
         volume:
           subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
 
   sparkd-ping-test:
     depends_on:
@@ -104,11 +104,94 @@ services:
     build:
       context: .
       dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
+        FROM bluetradingsystems/linux_dotnet8
         USER "root"
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
 
+  pubsub:
+    volumes:
+      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
+    build:
+      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      args:
+        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
+        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
+    ports:
+      - 22019:22019
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
+        if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
+          mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
+          set -a
+          . /home/bts/.local/share/bts/pubsub/conf/.env
+          set +a
+        fi
+        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
+          set -a
+          . /home/bts/.local/share/bts/pubsub/conf/.env.local
+          set +a
+        fi
+        ./PubSubServer"
+    env_file:
+      - .env
+      - ${PUBSUB__CONFIG_DIR}/.env
+
+  md-publisher:
+    volumes:
+      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
+      - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
+    build:
+      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+      args:
+        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
+        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
+        if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
+          mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
+          set -a
+          . /home/bts/.local/share/bts/md_publisher/conf/.env
+          set +a
+        fi
+        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
+          set -a
+          . /home/bts/.local/share/bts/md_publisher/conf/.env.local
+          set +a
+        fi
+      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
+    env_file:
+      - .env
+      - ${MD_PUBLISHER__CONFIG_DIR}/.env
+
+  api-server:
+    depends_on:
+      - sparkd
+    ports:
+      - 8008:8008
+      - 9443:9443
+    build:
+      dockerfile: src/client/spark_api_server/Dockerfile
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
 volumes:
   sparkd_data:
     name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
+  pubsub_data:
+    name: "pubsub_22019"
+  md_publisher_data:
+    name: "md_publisher_22019"
+  historic_md:
+    name: "historic_md"

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet8
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile

@@ -55,14 +55,16 @@ RUN \
    clangd-$clang_major
 
 # CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
+RUN set -eux \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends ca-certificates curl gpg \
+  && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
+    | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
+  && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    > /etc/apt/sources.list.d/kitware.list \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends cmake \
+  && rm -rf /var/lib/apt/lists/*
 
 # Dotnet
 RUN apt-get update && apt-get install -y dotnet8
@@ -80,9 +82,14 @@ RUN mkdir -p /etc/apt/keyrings \
 # Emumba (deps)
 RUN apt-get update && apt-get -y install rapidjson-dev
 # Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
+ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
+RUN --mount=type=secret,id=bitbucket_access_token \
+  set -eux \
+    && apt-get update \
+    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
+    && curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
+    && apt-get install -y /tmp/emumba-deribit-connector.deb \
+    && rm -f /tmp/emumba-deribit-connector.deb
 
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -90,6 +97,18 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
+# Rakurai
+ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
+RUN --mount=type=secret,id=bitbucket_access_token \
+  set -eux \
+    && apt-get update \
+    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
+    && curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
+    # Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
+    # spdlog headers.
+    && apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
+    && rm -f /tmp/rakuirai.deb
+
 # Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \

requirements.txt

@@ -18,10 +18,10 @@ MarkupSafe==3.0.2
 mccabe==0.6.1
 numpy==2.2.6
 platformdirs==2.3.0
-protobuf==4.21.12
+protobuf==6.33.4
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
+pybind11==3.0.1
+pybind11-stubgen==2.5.5
 pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20

spark_client.sln

@@ -1,14 +1,15 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
 EndProject
@@ -26,114 +27,403 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataSt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{1A5B8C5D-BADC-457E-819F-0050C44FE4D3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{2A0FBDE0-D870-4567-85BD-F7793A5862C6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
+		MinSizeRel|Any CPU = MinSizeRel|Any CPU
+		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
+		Release|Any CPU = Release|Any CPU
+		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
+		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
+		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.Build.0 = Debug|Any CPU
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.ActiveCfg = Release|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.Build.0 = Release|Any CPU
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.Build.0 = Debug|Any CPU
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.Build.0 = Release|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.ActiveCfg = Release|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.Build.0 = Release|Any CPU
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.Build.0 = Debug|Any CPU
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.ActiveCfg = Release|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.Build.0 = Release|Any CPU
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.Build.0 = Debug|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.ActiveCfg = Release|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.Build.0 = Release|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.Build.0 = Debug|Any CPU
 		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
 		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.Build.0 = Release|Any CPU
 		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.Build.0 = Debug|Any CPU
 		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
 		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.Build.0 = Release|Any CPU
 		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.Build.0 = Debug|Any CPU
 		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
 		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.Build.0 = Release|Any CPU
 		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.Build.0 = Debug|Any CPU
 		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
 		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.Build.0 = Release|Any CPU
 		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.Build.0 = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.Build.0 = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.ActiveCfg = Release|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.Build.0 = Release|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.ActiveCfg = Release|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.Build.0 = Release|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.ActiveCfg = Release|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.Build.0 = Release|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.Build.0 = Release|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.ActiveCfg = Release|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.Build.0 = Release|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.ActiveCfg = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.Build.0 = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.ActiveCfg = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.Build.0 = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{1A5B8C5D-BADC-457E-819F-0050C44FE4D3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6} = {1A5B8C5D-BADC-457E-819F-0050C44FE4D3}
+	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
 	EndGlobalSection

src/DataStreamSuite/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+   "dotnet.preferCSharpExtension": true
+}

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionGrammar.y

@@ -8,7 +8,6 @@
    public int Integer;
    public double Real;
    public string Identifier;
-   public DateTime DateTime; 
    public ImmutableList<IArithmeticExpression> Arguments;
    public ImmutableList<Assignment> Assignments;
    public IArithmeticExpression Argument;

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -11,10 +11,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ArithmeticExpression">
     <Scanner Include="ArithmeticExpressionTokens.lex" />

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -5,9 +5,5 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
+  
 </Project>

src/DataStreamSuite/BlueRegistry/DoubleRegistryHandler.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.Versioning;
+
 using Microsoft.Win32;
 
 namespace BCG
@@ -10,6 +12,7 @@ namespace BCG
          rk.SetValue (name, value.ToString ());
       }
 
+      [SupportedOSPlatform("windows")]
       public static double getValue (RegistryKey rk, string name)
       {
          object v = rk.GetValue (name);

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,23 +6,80 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var cmp1 = value1.Cast<IComparable>();
-      var cmp2 = value2.Cast<IComparable>();
+      var obj1 = value1.GetValue(env);
+      var obj2 = value2.GetValue(env);
+      if (obj1 is null) {
+         if (obj2 is null) {
+            return 0;
+         }
+         return -1;
+      }
+      if (obj2 is null) {
+         return 1;
+      }
+      return compareObjs(obj1, obj2);
+   }
+
+   static int compareObjs(object obj1, object obj2) {
       try {
-         var v1 = cmp1.GetValue(env);
-         var v2 = cmp2.GetValue(env);
-         return v1.GetType().IsNumeric() ?
-            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
+         var typ1 = obj1.GetType();
+         var typ2 = obj2.GetType();
+         bool areSameTypes = typ1 == typ2;
+         if (areSameTypes) {
+            return ((IComparable)obj1).CompareTo(obj2);
+         }
+         if (typ1.IsNumeric()) {
+            return tryCompareNumeric(obj1, obj2);
+         } else if (typ2.IsNumeric()) {
+            return -tryCompareNumeric(obj2, obj1);
+         } else if (typ1.IsEnum) {
+            if (typ2 == typeof(string)) {
+               return -((string)obj2).CompareTo(obj1.ToString());
+            }
+         } else if (typ1 == typeof(string)) {
+            return ((string)obj1).CompareTo(obj2.ToString());
+         } else if (typ1 == typeof(DateTime)) {
+            if (typ2 == typeof(TimeSpan)) {
+               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
+            }
+         } else if (typ1 == typeof(TimeSpan)) {
+            if (typ2 == typeof(DateTime)) {
+               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
+            }
+         }
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
+      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
+   }
+
+   static int tryCompareNumeric(object numericObj1, object obj2) {
+      var typ2 = obj2.GetType();
+      if (typ2.IsNumeric()) {
+         return new NumericComparer().Compare(numericObj1, obj2);
+      } else if (typ2.IsEnum) {
+         var undTyp2 = typ2.GetEnumUnderlyingType();
+         if (undTyp2 == typeof(byte)) {
+            return new NumericComparer().Compare(numericObj1, (byte)obj2);
+         }
+         if (undTyp2 == typeof(short)) {
+            return new NumericComparer().Compare(numericObj1, (short)obj2);
+         }
+         if (undTyp2 == typeof(int)) {
+            return new NumericComparer().Compare(numericObj1, (int)obj2);
+         }
+         if (undTyp2 == typeof(long)) {
+            return new NumericComparer().Compare(numericObj1, (long)obj2);
+         }
+      }
+      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpression.cs

@@ -1,19 +1,11 @@
 namespace Bts.BooleanExpressions;
 
-using bts;
-
 class BooleanExpression : IBooleanExpression {
    readonly Func<IDictionary<string, object?>, bool> evaluate;
 
    public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
       this.evaluate = evaluate;
 
-   public bool Evaluate(IDictionary<string, object?> env) {
-      try {
-         return evaluate(env);
-      } catch (Exception ex) {
-         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
-         throw;
-      }
-   }
+   public bool Evaluate(IDictionary<string, object?> env) =>
+      evaluate(env);
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -47,11 +47,6 @@ IsOfType                 '/'
 
 %%
 
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {BooleanConstant} {
    yylval.Boolean = yytext.ToLower() switch {
       "true" => true,
@@ -62,6 +57,11 @@ IsOfType                 '/'
    return (int)Token.BOOLEAN_CONSTANT;
 }
 
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {NullConstant} {
    yylval.Null = DBNull.Value;
    return (int)Token.NULL_CONSTANT;

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -11,10 +11,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
   
   <ItemGroup>
     <Scanner Include="BooleanExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -10,11 +10,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnExpression">
     <Scanner Include="ColumnExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -10,11 +10,20 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnSortExpression">
     <Scanner Include="ColumnSortExpressionTokens.lex" />

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -7,15 +7,21 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 
   <ItemGroup Label="ColumnMapExpression">
     <Scanner Include="ColumnMapExpressionTokens.lex" />

src/DataStreamSuite/DSPublisher/Configurations/.gitkeep

@@ -0,0 +1 @@
+

src/DataStreamSuite/DSPublisher/DSLWSClient/dsl_websocket_client.py

@@ -0,0 +1,168 @@
+"""
+Simple DSPublisher DSL websocket client.
+
+Example:
+   python dsl_websocket_client.py --dsl "HeikinAshiStream(...)"
+   python dsl_websocket_client.py --dsl-file query.dsl
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+try:
+   import websockets
+except ImportError:
+   print("Installing missing dependency: websockets")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
+   import websockets
+
+
+DEFAULT_DSL = 'RSIUIStream("ES", "TIME", 5, 7)'
+DEFAULT_URL = "ws://localhost:19093/ws/"
+
+
+def load_dsl(dsl: str | None, dsl_file: str | None) -> str:
+   if dsl and dsl_file:
+      raise ValueError("Pass either --dsl or --dsl-file, not both.")
+
+   if dsl_file:
+      return Path(dsl_file).read_text(encoding="utf-8").strip()
+
+   if dsl:
+      return dsl.strip()
+
+   if DEFAULT_DSL.startswith("#"):
+      raise ValueError("No DSL provided. Fill in DEFAULT_DSL, or pass --dsl / --dsl-file.")
+
+   return DEFAULT_DSL
+
+
+def load_config_path(config_path: str | None) -> Path | None:
+   if config_path:
+      return Path(config_path)
+
+   environment = os.environ.get("DOTNET_ENVIRONMENT")
+   config_file_name = "appsettings.json" if not environment else f"appsettings.{environment}.json"
+   script_dir = Path(__file__).resolve().parent
+   candidate_dirs = [
+      script_dir,
+      script_dir.parent,
+      Path.cwd(),
+   ]
+
+   for candidate_dir in candidate_dirs:
+      candidate_path = candidate_dir / config_file_name
+      if candidate_path.exists():
+         return candidate_path
+
+   return None
+
+
+def load_url(url: str | None, config_path: str | None) -> str:
+   if url:
+      return url
+
+   resolved_config_path = load_config_path(config_path)
+   if not resolved_config_path:
+      print(f"Config file not found; using default url: {DEFAULT_URL}")
+      return DEFAULT_URL
+
+   config = json.loads(resolved_config_path.read_text(encoding="utf-8"))
+   dsl_api_server_config = config.get("DSLAPIServer", {})
+   ip = dsl_api_server_config.get("Ip", "localhost")
+   port = int(dsl_api_server_config.get("Port", 19093))
+   ws_url = f"ws://{ip}:{port}/ws/"
+   print(f"Loaded DSPublisher DSL websocket url from {resolved_config_path}: {ws_url}")
+   return ws_url
+
+
+async def send_subscription(websocket, dsl: str) -> None:
+   payload = {
+      "type": "subscribe",
+      "dsl": dsl,
+   }
+   await websocket.send(json.dumps(payload))
+   print("Sent subscribe request:")
+   print(json.dumps(payload, indent=2))
+
+
+async def send_unsubscribe(websocket, subscription_id: str) -> None:
+   payload = {
+      "type": "unsubscribe",
+      "subscriptionId": subscription_id,
+   }
+   await websocket.send(json.dumps(payload))
+   print("Sent unsubscribe request:")
+   print(json.dumps(payload, indent=2))
+
+
+def print_server_message(raw_message: str) -> str | None:
+   try:
+      message = json.loads(raw_message)
+   except json.JSONDecodeError:
+      print(raw_message)
+      return None
+
+   print("Received server message:")
+   print(json.dumps(message, indent=2))
+   return message.get("subscriptionId")
+
+
+async def run_client(url: str, dsl: str) -> None:
+   subscription_id: str | None = None
+
+   async with websockets.connect(url) as websocket:
+      print(f"Connected to DSPublisher DSL websocket: {url}")
+      await send_subscription(websocket, dsl)
+
+      try:
+         async for raw_message in websocket:
+            latest_subscription_id = print_server_message(raw_message)
+            subscription_id = subscription_id or latest_subscription_id
+      finally:
+         if subscription_id:
+            await send_unsubscribe(websocket, subscription_id)
+
+
+def main() -> int:
+   parser = argparse.ArgumentParser(
+      description="Subscribe to a DSPublisher DSL websocket query and print responses."
+   )
+   parser.add_argument(
+      "--url",
+      help="DSPublisher DSL websocket URL. Overrides appsettings DSLAPIServer config.",
+   )
+   parser.add_argument(
+      "--config",
+      help="Path to DSPublisher appsettings JSON. Defaults to appsettings[.$DOTNET_ENVIRONMENT].json near this script.",
+   )
+   parser.add_argument(
+      "--dsl",
+      help="DSL expression to subscribe to.",
+   )
+   parser.add_argument(
+      "--dsl-file",
+      help="Path to a text file containing the DSL expression.",
+   )
+   args = parser.parse_args()
+
+   try:
+      dsl = load_dsl(args.dsl, args.dsl_file)
+      url = load_url(args.url, args.config)
+      asyncio.run(run_client(url, dsl))
+      return 0
+   except KeyboardInterrupt:
+      print("stopped")
+      return 0
+   except Exception as ex:
+      print(f"error: {ex}")
+      return 1
+
+
+if __name__ == "__main__":
+   raise SystemExit(main())

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLAPIWSListener.cs

@@ -0,0 +1,264 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Net;
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+using BCG;
+using Bts.DataStreamCore;
+
+/// <summary>
+/// Websocket-facing API boundary for DSL clients. It opens the listener port,
+/// accepts subscribe/unsubscribe messages, and sends DataPacket JSON back to clients.
+/// Instantiated by DSPAPIServer.
+/// </summary>
+public sealed class DSLAPIWSListener : IDisposable {
+   readonly HttpListener listener = new();
+   readonly IDSLBookkeeper bookkeeper;
+   readonly string prefix;
+   readonly ConcurrentDictionary<string, ClientConnection> clients_by_id = new();
+   CancellationTokenSource? cancellation_token_source;
+   Task? listener_task;
+   bool disposed;
+
+   sealed class ClientConnection {
+      public required string client_id { get; init; }
+      public required WebSocket web_socket { get; init; }
+      public HashSet<Guid> subscription_ids { get; } = new();
+      public SemaphoreSlim send_lock { get; } = new(1, 1);
+      public DataPacketJsonSerializer serializer { get; } = new();
+   }
+
+   public DSLAPIWSListener(string prefix, IDSLBookkeeper bookkeeper) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(prefix);
+      ArgumentNullException.ThrowIfNull(bookkeeper);
+      this.prefix = prefix;
+      this.bookkeeper = bookkeeper;
+   }
+
+   public Task StartAsync(CancellationToken cancellation_token = default) {
+      _throw_if_disposed();
+      if (listener_task is not null) {
+         return Task.CompletedTask;
+      }
+
+      cancellation_token_source = CancellationTokenSource.CreateLinkedTokenSource(cancellation_token);
+      listener.Prefixes.Add(prefix);
+      listener.Start();
+      listener_task = Task.Run(() => _listen_loop(cancellation_token_source.Token), CancellationToken.None);
+      return Task.CompletedTask;
+   }
+
+   public async Task StopAsync() {
+      if (cancellation_token_source is null) {
+         return;
+      }
+
+      cancellation_token_source.Cancel();
+      if (listener.IsListening) {
+         listener.Stop();
+      }
+
+      foreach (var client_connection in clients_by_id.Values) {
+         await _cleanup_client(client_connection);
+      }
+
+      if (listener_task is not null) {
+         await listener_task;
+         listener_task = null;
+      }
+
+      cancellation_token_source.Dispose();
+      cancellation_token_source = null;
+   }
+
+   public void Dispose() {
+      if (disposed) {
+         return;
+      }
+
+      StopAsync().GetAwaiter().GetResult();
+      listener.Close();
+      disposed = true;
+   }
+
+   async Task _listen_loop(CancellationToken cancellation_token) {
+      while (!cancellation_token.IsCancellationRequested) {
+         HttpListenerContext context;
+         try {
+            context = await listener.GetContextAsync();
+         } catch (HttpListenerException) when (cancellation_token.IsCancellationRequested || !listener.IsListening) {
+            break;
+         } catch (ObjectDisposedException) {
+            break;
+         }
+
+         _ = Task.Run(() => _handle_request(context, cancellation_token), cancellation_token);
+      }
+   }
+
+   async Task _handle_request(HttpListenerContext context, CancellationToken cancellation_token) {
+      if (!context.Request.IsWebSocketRequest) {
+         context.Response.StatusCode = (int)HttpStatusCode.BadRequest;
+         context.Response.Close();
+         return;
+      }
+
+      HttpListenerWebSocketContext web_socket_context = await context.AcceptWebSocketAsync(null);
+      var client_connection = new ClientConnection {
+         client_id = Guid.NewGuid().ToString("N"),
+         web_socket = web_socket_context.WebSocket,
+      };
+
+      clients_by_id[client_connection.client_id] = client_connection;
+      try {
+         await _read_messages(client_connection, cancellation_token);
+      } finally {
+         await _cleanup_client(client_connection);
+      }
+   }
+
+   async Task _read_messages(ClientConnection client_connection, CancellationToken cancellation_token) {
+      var buffer = new byte[64 * 1024];
+      while (client_connection.web_socket.State == WebSocketState.Open && !cancellation_token.IsCancellationRequested) {
+         var result = await client_connection.web_socket.ReceiveAsync(buffer, cancellation_token);
+         if (result.MessageType == WebSocketMessageType.Close) {
+            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "ok", cancellation_token);
+            break;
+         }
+
+         var json = Encoding.UTF8.GetString(buffer, 0, result.Count);
+         await _handle_message(client_connection, json, cancellation_token);
+      }
+   }
+
+   async Task _handle_message(ClientConnection client_connection, string json, CancellationToken cancellation_token) {
+      DSLWebSocketRequest? request;
+      try {
+         request = JsonSerializer.Deserialize<DSLWebSocketRequest>(json);
+      } catch (JsonException ex) {
+         await _send_error(client_connection, $"Invalid request JSON: {ex.Message}", cancellation_token);
+         return;
+      }
+
+      if (request?.type is null) {
+         await _send_error(client_connection, "Request type is required.", cancellation_token);
+         return;
+      }
+
+      if (string.Equals(request.type, "subscribe", StringComparison.OrdinalIgnoreCase)) {
+         await _handle_subscribe(client_connection, request, cancellation_token);
+         return;
+      }
+
+      if (string.Equals(request.type, "unsubscribe", StringComparison.OrdinalIgnoreCase)) {
+         await _handle_unsubscribe(client_connection, request, cancellation_token);
+         return;
+      }
+
+      await _send_error(client_connection, $"Unknown request type: {request.type}", cancellation_token);
+   }
+
+   async Task _handle_subscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
+      if (string.IsNullOrWhiteSpace(request.dsl)) {
+         await _send_error(client_connection, "Subscribe request requires dsl.", cancellation_token);
+         return;
+      }
+
+      Guid subscription_id = Guid.Empty;
+      subscription_id = bookkeeper.Subscribe(request.dsl, packet => {
+         if (subscription_id == Guid.Empty) {
+            return;
+         }
+         _ = _send_packet(client_connection, subscription_id, packet, CancellationToken.None);
+      }, client_connection.client_id);
+
+      client_connection.subscription_ids.Add(subscription_id);
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "subscribed",
+         subscription_id = subscription_id,
+      }, cancellation_token);
+   }
+
+   async Task _handle_unsubscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
+      if (request.subscription_id is null) {
+         await _send_error(client_connection, "Unsubscribe request requires subscriptionId.", cancellation_token);
+         return;
+      }
+
+      var subscription_id = request.subscription_id.Value;
+      if (!client_connection.subscription_ids.Remove(subscription_id)) {
+         await _send_error(client_connection, $"Subscription does not belong to this client: {subscription_id}", cancellation_token);
+         return;
+      }
+
+      bookkeeper.Unsubscribe(subscription_id);
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "unsubscribed",
+         subscription_id = subscription_id,
+      }, cancellation_token);
+   }
+
+   async Task _send_packet(ClientConnection client_connection, Guid subscription_id, DataPacket packet, CancellationToken cancellation_token) {
+      var packet_json = client_connection.serializer.Serialize(packet);
+      using var packet_document = JsonDocument.Parse(packet_json ?? "{}");
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "data",
+         subscription_id = subscription_id,
+         packet = packet_document.RootElement.Clone(),
+      }, cancellation_token);
+   }
+
+   Task _send_error(ClientConnection client_connection, string message, CancellationToken cancellation_token) =>
+      _send_response(client_connection, new DSLWebSocketResponse {
+         type = "error",
+         message = message,
+      }, cancellation_token);
+
+   async Task _send_response(ClientConnection client_connection, DSLWebSocketResponse response, CancellationToken cancellation_token) {
+      if (client_connection.web_socket.State != WebSocketState.Open) {
+         bookkeeper.DisposeClient(client_connection.client_id);
+         return;
+      }
+
+      var json = JsonSerializer.Serialize(response);
+      var bytes = Encoding.UTF8.GetBytes(json);
+      await client_connection.send_lock.WaitAsync(cancellation_token);
+      try {
+         await client_connection.web_socket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellation_token);
+      } catch {
+         bookkeeper.DisposeClient(client_connection.client_id);
+         throw;
+      } finally {
+         client_connection.send_lock.Release();
+      }
+   }
+
+   async Task _cleanup_client(ClientConnection client_connection) {
+      if (clients_by_id.TryRemove(client_connection.client_id, out _)) {
+         bookkeeper.DisposeClient(client_connection.client_id);
+      }
+
+      if (client_connection.web_socket.State == WebSocketState.Open || client_connection.web_socket.State == WebSocketState.CloseReceived) {
+         try {
+            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "cleanup", CancellationToken.None);
+         } catch {
+         }
+      }
+
+      client_connection.send_lock.Dispose();
+      client_connection.web_socket.Dispose();
+   }
+
+   void _throw_if_disposed() {
+      if (disposed) {
+         throw new ObjectDisposedException(nameof(DSLAPIWSListener));
+      }
+   }
+}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLBookkeeper.cs

@@ -0,0 +1,216 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Collections.Generic;
+
+using BCG;
+using Bts.DataStreamCore;
+using bts.BlueOrb;
+using bts;
+
+/// <summary>
+/// Tracks active DSL subscriptions and shared DSLStream instances. Multiple clients
+/// subscribing to the same DSL reuse one stream, and DoWork dispatches queued packets.
+/// Instantiated by DSPAPIServer.
+/// </summary>
+public sealed class DSLBookkeeper : IDSLBookkeeper {
+   sealed class QueryState {
+      public required string dsl { get; init; }
+      public required DSLStream stream { get; init; }
+      public DataStream.ChangeDelegate? stream_handler { get; set; }
+      public HashSet<Guid> subscription_ids { get; } = new();
+      public Queue<DataPacket> pending_packets { get; } = new();
+   }
+
+   sealed class SubscriptionState {
+      public required Guid subscription_id { get; init; }
+      public required string client_id { get; init; }
+      public required string dsl_key { get; init; }
+      public required Action<DataPacket> on_data { get; init; }
+   }
+
+   readonly object lock_obj = new();
+   readonly Dictionary<string, QueryState> queries_by_dsl = new(StringComparer.Ordinal);
+   readonly Dictionary<Guid, SubscriptionState> subscriptions_by_id = new();
+   readonly Dictionary<string, HashSet<Guid>> subscriptions_by_client = new(StringComparer.Ordinal);
+   bool disposed;
+
+   public Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(dsl);
+      ArgumentNullException.ThrowIfNull(on_data);
+      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
+
+      lock (lock_obj) {
+         _throw_if_disposed();
+
+         var dsl_key = _normalize_dsl(dsl);
+         if (!queries_by_dsl.TryGetValue(dsl_key, out var query_state)) {
+            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} creating new DSLStream dsl: {dsl_key}");
+            query_state = _create_query_state(dsl_key);
+            queries_by_dsl.Add(dsl_key, query_state);
+         } else {
+            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} reusing existing DSLStream subscription_count: {query_state.subscription_ids.Count} dsl: {dsl_key}");
+         }
+
+         var subscription_id = Guid.NewGuid();
+         var subscription_state = new SubscriptionState {
+            subscription_id = subscription_id,
+            client_id = client_id,
+            dsl_key = dsl_key,
+            on_data = on_data,
+         };
+
+         subscriptions_by_id.Add(subscription_id, subscription_state);
+         query_state.subscription_ids.Add(subscription_id);
+
+         if (!subscriptions_by_client.TryGetValue(client_id, out var client_subscriptions)) {
+            client_subscriptions = new();
+            subscriptions_by_client.Add(client_id, client_subscriptions);
+         }
+         client_subscriptions.Add(subscription_id);
+
+         _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} subscription_id: {subscription_id} active_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {dsl_key}");
+         return subscription_id;
+      }
+   }
+
+   public void Unsubscribe(Guid subscription_id) {
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+         _unsubscribe_locked(subscription_id);
+      }
+   }
+
+   public void DisposeClient(string client_id) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
+
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         if (!subscriptions_by_client.TryGetValue(client_id, out var subscription_ids)) {
+            return;
+         }
+
+         foreach (var subscription_id in subscription_ids.ToArray()) {
+            _unsubscribe_locked(subscription_id);
+         }
+      }
+   }
+
+   public void DoWork() {
+      List<(Action<DataPacket> on_data, DataPacket packet)> callbacks = new();
+
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         foreach (var query_state in queries_by_dsl.Values) {
+            while (query_state.pending_packets.TryDequeue(out var packet)) {
+               foreach (var subscription_id in query_state.subscription_ids) {
+                  if (subscriptions_by_id.TryGetValue(subscription_id, out var subscription_state)) {
+                     callbacks.Add((subscription_state.on_data, packet));
+                  }
+               }
+            }
+         }
+      }
+
+      foreach (var callback in callbacks) {
+         callback.on_data(callback.packet);
+      }
+   }
+
+   public void Dispose() {
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         foreach (var query_state in queries_by_dsl.Values) {
+            _dispose_query_state(query_state);
+         }
+
+         queries_by_dsl.Clear();
+         subscriptions_by_id.Clear();
+         subscriptions_by_client.Clear();
+         disposed = true;
+      }
+   }
+
+   QueryState _create_query_state(string dsl_key) {
+      _log_lifecycle($"DSLBookkeeper._create_query_state creating DSLStream dsl: {dsl_key}");
+      var stream = new DSLStream(dsl_key);
+      var query_state = new QueryState {
+         dsl = dsl_key,
+         stream = stream,
+      };
+
+      DataStream.ChangeDelegate stream_handler = new_value => {
+         if (new_value is not DataPacket packet) {
+            return;
+         }
+
+         lock (lock_obj) {
+            if (!disposed) {
+               query_state.pending_packets.Enqueue(packet);
+            }
+         }
+      };
+
+      query_state.stream_handler = stream_handler;
+      stream.onChange += stream_handler;
+      return query_state;
+   }
+
+   void _unsubscribe_locked(Guid subscription_id) {
+      if (!subscriptions_by_id.Remove(subscription_id, out var subscription_state)) {
+         return;
+      }
+
+      if (subscriptions_by_client.TryGetValue(subscription_state.client_id, out var client_subscriptions)) {
+         client_subscriptions.Remove(subscription_id);
+         if (client_subscriptions.Count == 0) {
+            subscriptions_by_client.Remove(subscription_state.client_id);
+         }
+      }
+
+      if (queries_by_dsl.TryGetValue(subscription_state.dsl_key, out var query_state)) {
+         query_state.subscription_ids.Remove(subscription_id);
+         _log_lifecycle($"DSLBookkeeper.Unsubscribe subscription_id: {subscription_id} client_id: {subscription_state.client_id} remaining_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {subscription_state.dsl_key}");
+         if (query_state.subscription_ids.Count == 0) {
+            _log_lifecycle($"DSLBookkeeper.Unsubscribe garbage collecting DSLStream; last subscription removed dsl: {subscription_state.dsl_key}");
+            _dispose_query_state(query_state);
+            queries_by_dsl.Remove(subscription_state.dsl_key);
+         }
+      }
+   }
+
+   static string _normalize_dsl(string dsl) =>
+      dsl.Trim();
+
+   static void _dispose_query_state(QueryState query_state) {
+      _log_lifecycle($"DSLBookkeeper._dispose_query_state disposing DSLStream dsl: {query_state.dsl} pending_packets: {query_state.pending_packets.Count} subscription_count: {query_state.subscription_ids.Count}");
+      if (query_state.stream_handler is not null) {
+         query_state.stream.onChange -= query_state.stream_handler;
+         query_state.stream_handler = null;
+      }
+      query_state.subscription_ids.Clear();
+      query_state.pending_packets.Clear();
+   }
+
+   static void _log_lifecycle(string message) {
+      Console.WriteLine(message);
+      Log.Gen.info(message);
+   }
+
+   void _throw_if_disposed() {
+      if (disposed) {
+         throw new ObjectDisposedException(nameof(DSLBookkeeper));
+      }
+   }
+}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLWebSocketMessages.cs

@@ -0,0 +1,39 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+/// <summary>
+/// Client-to-server websocket protocol message for subscribing and unsubscribing.
+/// Instantiated by System.Text.Json when DSLAPIWSListener deserializes client messages.
+/// </summary>
+public sealed class DSLWebSocketRequest {
+   [JsonPropertyName("type")]
+   public string? type { get; set; }
+
+   [JsonPropertyName("dsl")]
+   public string? dsl { get; set; }
+
+   [JsonPropertyName("subscriptionId")]
+   public Guid? subscription_id { get; set; }
+}
+
+/// <summary>
+/// Server-to-client websocket protocol message. Data responses carry serialized
+/// DataPacket JSON in the packet field.
+/// Instantiated by DSLAPIWSListener before serializing responses to clients.
+/// </summary>
+public sealed class DSLWebSocketResponse {
+   [JsonPropertyName("type")]
+   public string type { get; set; } = "";
+
+   [JsonPropertyName("subscriptionId")]
+   public Guid? subscription_id { get; set; }
+
+   [JsonPropertyName("message")]
+   public string? message { get; set; }
+
+   [JsonPropertyName("packet")]
+   public JsonElement? packet { get; set; }
+}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSPAPIServer.cs

@@ -0,0 +1,37 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+/// <summary>
+/// Top-level DSPublisher API server. Owns the websocket listener and the DSL
+/// subscription bookkeeper, and exposes the small lifecycle surface Program.cs uses.
+/// Instantiated by Program.cs.
+/// </summary>
+public sealed class DSPAPIServer : IDisposable {
+   readonly DSLBookkeeper bookkeeper;
+   readonly DSLAPIWSListener web_socket_listener;
+
+   public DSPAPIServer(global::DSLAPIServerConfig config) {
+      ArgumentNullException.ThrowIfNull(config);
+      bookkeeper = new DSLBookkeeper();
+      var web_socket_prefix = $"http://{config.Ip}:{config.Port}/ws/";
+      web_socket_listener = new DSLAPIWSListener(web_socket_prefix, bookkeeper);
+      Console.WriteLine($"DSPAPIServer listening for websocket DSL API requests on {web_socket_prefix}");
+   }
+
+   public Task StartAsync(CancellationToken cancellation_token = default) =>
+      web_socket_listener.StartAsync(cancellation_token);
+
+   public Task StopAsync() =>
+      web_socket_listener.StopAsync();
+
+   public void DoWork() =>
+      bookkeeper.DoWork();
+
+   public void Dispose() {
+      web_socket_listener.Dispose();
+      bookkeeper.Dispose();
+   }
+}

src/DataStreamSuite/DSPublisher/DSLWSServer/IDSLBookkeeper.cs

@@ -0,0 +1,18 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+
+using BCG;
+using Bts.DataStreamCore;
+
+/// <summary>
+/// Internal contract between the websocket listener and the DSL subscription owner.
+/// The listener handles transport; the bookkeeper owns stream lifetime and callbacks.
+/// Implemented by DSLBookkeeper and consumed by DSLAPIWSListener.
+/// </summary>
+public interface IDSLBookkeeper : IDisposable {
+   Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id);
+   void Unsubscribe(Guid subscription_id);
+   void DisposeClient(string client_id);
+   void DoWork();
+}

src/DataStreamSuite/DSPublisher/DSPublisher.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\PubSub\PubSub.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ConfigurationProfile Include="appsettings*.json" />
+
+    <None Update="appsettings*.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <TargetPath>DSPublisher.%(Filename)%(Extension)</TargetPath>
+    </None>
+    <None Update="WSPublishers\**\*.py">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="DSLWSClient\**\*.py">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/DSPublisher/DSPublisherConfig.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+
+public sealed class DSPublisherConfig {
+   public List<StreamPublishingNode> StreamPublishingNodes { get; init; } = new();
+   public List<PythonPublishingNode> PythonPublishingNodes { get; init; } = new();
+}
+
+public sealed class DSLAPIServerConfig {
+   public string Ip { get; init; } = "localhost";
+   public int Port { get; init; } = 19093;
+}
+
+public sealed class StreamPublishingNode {
+   public string Name { get; init; } = string.Empty;
+   public string Expression { get; init; } = string.Empty;
+}
+
+public sealed class PythonPublishingNode {
+   public string PythonScriptLocation { get; init; } = string.Empty;
+   public string Args { get; init; } = string.Empty;
+}

src/DataStreamSuite/DSPublisher/Program.cs

@@ -0,0 +1,125 @@
+using System;
+using System.IO;
+using System.Text.Json;
+using System.Threading;
+
+using BCG;
+using bts;
+using Bts.PubSub;
+using bts.BlueOrb;
+using DSPublisher.DSLWSServer;
+
+Console.WriteLine("Starting DSPublisher");
+
+var configPath = load_config_path();
+Console.WriteLine($"Loading config from: {configPath}");
+if (!File.Exists(configPath)) {
+   Console.WriteLine($"Config file not found: {configPath}");
+   return;
+}
+
+var json = File.ReadAllText(configPath);
+using var doc = JsonDocument.Parse(json);
+
+if (!doc.RootElement.TryGetProperty("DSPublisher", out var dspPublisherElement)) {
+   Console.WriteLine("DSPublisher config section not found");
+   return;
+}
+
+var json_options = new JsonSerializerOptions {
+   PropertyNameCaseInsensitive = true,
+};
+
+var config = dspPublisherElement.Deserialize<DSPublisherConfig>(json_options) ?? new DSPublisherConfig();
+var dsl_api_server_config = load_dsl_api_server_config(doc, json_options);
+
+Console.WriteLine("StreamPublishingNodes:");
+foreach (var node in config.StreamPublishingNodes) {
+   Console.WriteLine($"  Name: {node.Name}");
+   Console.WriteLine($"  Expression: {node.Expression}");
+}
+
+Console.WriteLine("PythonPublishingNodes:");
+foreach (var node in config.PythonPublishingNodes) {
+   Console.WriteLine($"  PythonScriptLocation: {node.PythonScriptLocation}");
+   Console.WriteLine($"  Args: {node.Args}");
+}
+
+var pubsub_config = load_pubsub_config(doc, json_options);
+PubSubConfig.SetGlobalConfig(pubsub_config);
+using var view_stream_database = new ViewStreamDatabase(new SubscriberCoreFactory());
+ViewStreamDatabase.SetGlobalInstance(view_stream_database);
+
+//var pub_table_broker = new PubTableBroker();
+//var stream_publishers = _process_publishing_nodes(config.StreamPublishingNodes, pub_table_broker);
+
+using var dsp_api_server = new DSPAPIServer(dsl_api_server_config);
+await dsp_api_server.StartAsync();
+
+while (true) {
+   view_stream_database.doWork();
+   dsp_api_server.DoWork();
+   Thread.Sleep(TimeSpan.FromMilliseconds(10));
+}
+
+string load_config_path() {
+   var environment = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+   var baseFileName = string.IsNullOrWhiteSpace(environment)
+      ? "appsettings.json"
+      : $"appsettings.{environment}.json";
+   var dspPublisherFileName = $"DSPublisher.{baseFileName}";
+   var dspPublisherConfigPath = Path.Combine(AppContext.BaseDirectory, dspPublisherFileName);
+   if (File.Exists(dspPublisherConfigPath)) {
+      return dspPublisherConfigPath;
+   }
+
+   return Path.Combine(AppContext.BaseDirectory, baseFileName);
+}
+
+PubSubConfig load_pubsub_config(JsonDocument doc, JsonSerializerOptions json_options) {
+   if (!doc.RootElement.TryGetProperty("PubSub", out var pubsub_element)) {
+      Log.Gen.warn("PubSub config section not found; using PubSubConfig.Default().");
+      return PubSubConfig.Default();
+   }
+
+   var pubsub_config = pubsub_element.Deserialize<PubSubConfig>(json_options);
+   if (pubsub_config is null) {
+      Log.Gen.error("PubSub config section could not be deserialized; using PubSubConfig.Default().");
+      return PubSubConfig.Default();
+   }
+
+   Log.Gen.info("Loaded PubSub config from DSPublisher config file.");
+   return pubsub_config;
+}
+
+DSLAPIServerConfig load_dsl_api_server_config(JsonDocument doc, JsonSerializerOptions json_options) {
+   if (!doc.RootElement.TryGetProperty("DSLAPIServer", out var dsl_api_server_element)) {
+      Log.Gen.warn("DSLAPIServer config section not found; using DSLAPIServerConfig defaults.");
+      return new DSLAPIServerConfig();
+   }
+
+   var dsl_api_server_config = dsl_api_server_element.Deserialize<DSLAPIServerConfig>(json_options);
+   if (dsl_api_server_config is null) {
+      Log.Gen.error("DSLAPIServer config section could not be deserialized; using DSLAPIServerConfig defaults.");
+      return new DSLAPIServerConfig();
+   }
+
+   Log.Gen.info($"Loaded DSLAPIServer config ip: {dsl_api_server_config.Ip} port: {dsl_api_server_config.Port}");
+   return dsl_api_server_config;
+}
+
+List<IDisposable> _process_publishing_nodes(IEnumerable<StreamPublishingNode> publishing_nodes, PubTableBroker pub_table_broker) {
+   var stream_publishers = new List<IDisposable>();
+   foreach (var node in publishing_nodes) {
+      if (string.IsNullOrWhiteSpace(node.Name) || string.IsNullOrWhiteSpace(node.Expression)) {
+         Log.Gen.error("Skipping StreamPublishingNode because name or expression is empty.");
+         continue;
+      }
+
+      Log.Gen.info($"Creating ViewPublisher name: {node.Name} expression: {node.Expression}");
+      var stream = new DSLStream(node.Expression);
+      stream_publishers.Add(new ViewPublisher(node.Name, pub_table_broker, stream));
+   }
+
+   return stream_publishers;
+}

src/DataStreamSuite/DSPublisher/Services/.gitkeep

@@ -0,0 +1 @@
+

src/DataStreamSuite/DSPublisher/WSPublishers/DatabentoPublisher.py

@@ -0,0 +1,146 @@
+"""
+Requires:
+   pip install -U databento
+   pip install websockets
+
+Sources:
+   https://databento.com/docs/quickstart/set-up
+   https://databento.com/docs/schemas-and-data-formats/bbo
+"""
+
+import datetime as dt
+import subprocess
+import sys
+import time
+
+try:
+   import databento as db
+except ImportError:
+   print("Installing missing dependency: databento")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
+   import databento as db
+
+from databento.common.error import BentoError
+
+try:
+   import websockets
+except ImportError:
+   print("Installing missing dependency: websockets")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
+   import websockets
+
+from databento_transformer import DatabentoTransformer
+from pubsub_client import PubSubClient
+
+
+API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
+DATASET = "GLBX.MDP3"
+SYMBOL = "ES.FUT"
+PUBSUB_WS_URL = "ws://localhost:19092/ws/"
+BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
+TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
+DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
+
+_pubsub_client = PubSubClient(PUBSUB_WS_URL)
+_transformer = DatabentoTransformer(
+   symbol=SYMBOL,
+   bbo_table_name=BBO_TABLE_NAME,
+   trades_table_name=TRADES_TABLE_NAME,
+   definitions_table_name=DEFINITIONS_TABLE_NAME,
+)
+
+
+def _publish_contract_definitions() -> None:
+   historical_client = db.Historical(API_KEY)
+   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
+   schema_range = dataset_range.get("schema", {}).get("definition", {})
+   available_end = schema_range.get("end", dataset_range["end"])
+
+   if isinstance(available_end, str):
+      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
+   else:
+      available_end_dt = available_end
+
+   # Databento's range end is exclusive. Back off slightly so the request stays
+   # strictly inside the available window.
+   end_dt = available_end_dt - dt.timedelta(seconds=1)
+   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
+   start = start_dt.isoformat()
+   end = end_dt.isoformat()
+
+   definition_data = historical_client.timeseries.get_range(
+      dataset=DATASET,
+      schema="definition",
+      stype_in="parent",
+      symbols=SYMBOL,
+      start=start,
+      end=end,
+   )
+
+   df = definition_data.to_df()
+   payload = _transformer.build_definitions_payload(df)
+   if payload is None:
+      print("No Databento contract definitions returned")
+      return
+
+   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
+   _pubsub_client.publish(payload)
+
+
+def on_record(record) -> None:
+   # Print every callback first so we can see whether Databento is flowing
+   # even if our row extraction logic misses a field shape.
+   print(record)
+
+   payloads = _transformer.transform_record(record)
+   if payloads:
+      for payload in payloads:
+         _pubsub_client.publish(payload)
+      return
+
+   print("Skipping record with no BBO/trade fields")
+
+
+def main() -> None:
+   _pubsub_client.start()
+   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
+   try:
+      _publish_contract_definitions()
+   except Exception as ex:
+      print(f"Contract definition publish failed: {ex}", flush=True)
+
+   while True:
+      try:
+         client = db.Live(key=API_KEY)
+
+         client.subscribe(
+            dataset=DATASET,
+            schema="bbo-1s",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+         client.subscribe(
+            dataset=DATASET,
+            schema="trades",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+
+         client.add_callback(on_record)
+         client.start()
+         print("Databento client started, waiting for BBO and trade records...", flush=True)
+         client.block_for_close()
+         print("Databento client closed cleanly", flush=True)
+         break
+      except BentoError as ex:
+         print(f"Databento live connection error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+      except Exception as ex:
+         print(f"Databento publisher unexpected error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+
+
+if __name__ == "__main__":
+   main()

src/DataStreamSuite/DSPublisher/WSPublishers/databento_transformer.py

@@ -0,0 +1,244 @@
+import datetime as dt
+
+
+class DatabentoTransformer:
+   def __init__(
+      self,
+      symbol: str,
+      bbo_table_name: str,
+      trades_table_name: str,
+      definitions_table_name: str,
+   ):
+      self._symbol = symbol
+      self._bbo_table_name = bbo_table_name
+      self._trades_table_name = trades_table_name
+      self._definitions_table_name = definitions_table_name
+      self._instrument_id_to_symbol: dict[int, str] = {}
+      self._last_bbo_row: dict | None = None
+      self._trade_row_id = 0
+      self._trade_table_created = False
+
+   def transform_record(self, record) -> list[dict]:
+      payload = self._handle_bbo_record(record)
+      if payload is not None:
+         return [payload]
+
+      payload = self._handle_trade_record(record)
+      if payload is not None:
+         return [payload]
+
+      return []
+
+   def build_definitions_payload(self, definition_df) -> dict | None:
+      if definition_df is None or definition_df.empty:
+         return None
+
+      df = definition_df.reset_index()
+      if "instrument_class" in df.columns:
+         df = df[df["instrument_class"].astype(str) == "F"]
+      if "security_type" in df.columns:
+         df = df[df["security_type"].astype(str) == "FUT"]
+      if "raw_symbol" in df.columns:
+         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
+         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
+
+      rows: list[dict] = []
+      for idx, row in enumerate(df.itertuples(index=False), start=1):
+         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
+         security_name = str(getattr(row, "raw_symbol", ""))
+         if instrument_id > 0 and security_name:
+            self._instrument_id_to_symbol[instrument_id] = security_name
+
+         rows.append({
+            "id": idx,
+            "security_name": security_name,
+            "instrument_id": instrument_id,
+            "instrument_class": str(getattr(row, "instrument_class", "")),
+            "security_type": str(getattr(row, "security_type", "")),
+            "asset": str(getattr(row, "asset", "")),
+            "exchange": str(getattr(row, "exchange", "")),
+            "currency": str(getattr(row, "currency", "")),
+            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
+            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
+            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
+            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
+            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
+            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
+            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
+            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
+            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
+            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
+            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
+            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
+            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
+            "underlying": str(getattr(row, "underlying", "")),
+         })
+
+      if not rows:
+         return None
+
+      return {
+         "op": "create_table",
+         "table_name": self._definitions_table_name,
+         "rows": rows,
+      }
+
+   def _handle_bbo_record(self, record) -> dict | None:
+      row = self._to_bbo_row(record)
+      if row is None:
+         return None
+
+      if self._last_bbo_row is None:
+         payload = {
+            "op": "create_table",
+            "table_name": self._bbo_table_name,
+            "rows": [row],
+         }
+      else:
+         payload = {
+            "op": "multiple_remove_add",
+            "table_name": self._bbo_table_name,
+            "removed_rows": [self._last_bbo_row],
+            "added_rows": [row],
+         }
+
+      self._last_bbo_row = row
+      return payload
+
+   def _handle_trade_record(self, record) -> dict | None:
+      row = self._to_trade_row(record)
+      if row is None:
+         return None
+
+      row["id"] = self._trade_row_id
+      self._trade_row_id += 1
+
+      if not self._trade_table_created:
+         payload = {
+            "op": "create_table",
+            "table_name": self._trades_table_name,
+            "rows": [row],
+         }
+         self._trade_table_created = True
+         return payload
+
+      return {
+         "op": "multiple_remove_add",
+         "table_name": self._trades_table_name,
+         "removed_rows": [],
+         "added_rows": [row],
+      }
+
+   def _to_bbo_row(self, record) -> dict | None:
+      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
+      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
+      if bid_px is None and ask_px is None:
+         return None
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "bid_px": float(bid_px or 0.0),
+         "ask_px": float(ask_px or 0.0),
+         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
+         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
+      }
+
+   def _to_trade_row(self, record) -> dict | None:
+      price = self._get_attr(record, "price", "px")
+      size = self._get_attr(record, "size")
+      if price is None and size is None:
+         return None
+
+      side = self._get_attr(record, "side", "aggressor_side", default="")
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "price": float(price or 0.0),
+         "size": int(size or 0),
+         "side": str(side),
+      }
+
+   def _get_security_name(self, record) -> str:
+      instrument_id = self._get_attr(record, "instrument_id", default=0)
+      try:
+         instrument_id_int = int(instrument_id or 0)
+      except (TypeError, ValueError):
+         instrument_id_int = 0
+
+      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
+         return self._instrument_id_to_symbol[instrument_id_int]
+
+      value = self._get_attr(
+         record,
+         "symbol",
+         "raw_symbol",
+         "stype_out_symbol",
+         "stype_in_symbol",
+         default=self._symbol,
+      )
+      return str(value)
+
+   def _get_timestamp(self, record) -> int:
+      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
+      try:
+         return int(value)
+      except (TypeError, ValueError):
+         return 0
+
+   def _get_attr(self, record, *names, default=None):
+      for name in names:
+         if hasattr(record, name):
+            value = getattr(record, name)
+            if value is not None:
+               return value
+      return default
+
+   def _get_level_attr(self, record, name, default=None):
+      levels = self._get_attr(record, "levels", default=None)
+      if levels:
+         level0 = levels[0]
+         if hasattr(level0, name):
+            value = getattr(level0, name)
+            if value is not None:
+               return value
+      return default
+
+   def _ns_to_iso(self, timestamp_ns) -> str:
+      if timestamp_ns is None:
+         return ""
+
+      if hasattr(timestamp_ns, "isoformat"):
+         try:
+            if hasattr(timestamp_ns, "to_pydatetime"):
+               timestamp_dt = timestamp_ns.to_pydatetime()
+            else:
+               timestamp_dt = timestamp_ns
+
+            if timestamp_dt is None:
+               return ""
+
+            if getattr(timestamp_dt, "tzinfo", None) is None:
+               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
+            return timestamp_dt.isoformat()
+         except (TypeError, ValueError, AttributeError):
+            pass
+
+      try:
+         ts_int = int(timestamp_ns)
+         if ts_int <= 0:
+            return ""
+         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
+      except (TypeError, ValueError, OSError, OverflowError):
+         return ""
+
+   def _nano_to_float(self, value) -> float:
+      try:
+         return float(value) / 1_000_000_000.0
+      except (TypeError, ValueError):
+         return 0.0

src/DataStreamSuite/DSPublisher/WSPublishers/pubsub_client.py

@@ -0,0 +1,58 @@
+import asyncio
+import json
+import queue
+import threading
+
+import websockets
+
+
+class PubSubClient:
+   def __init__(self, ws_url: str):
+      self._ws_url = ws_url
+      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
+      self._sender_thread: threading.Thread | None = None
+
+   def start(self) -> threading.Thread:
+      if self._sender_thread is None or not self._sender_thread.is_alive():
+         self._sender_thread = self._start_sender_thread()
+      return self._sender_thread
+
+   def publish(self, payload: dict) -> None:
+      self._outgoing_queue.put(payload)
+
+   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
+      self.publish({
+         "op": "create_table",
+         "table_name": table_name,
+         "rows": rows,
+      })
+
+   def publish_multiple_remove_add(
+      self,
+      table_name: str,
+      removed_rows: list[dict],
+      added_rows: list[dict],
+   ) -> None:
+      self.publish({
+         "op": "multiple_remove_add",
+         "table_name": table_name,
+         "removed_rows": removed_rows,
+         "added_rows": added_rows,
+      })
+
+   def _start_sender_thread(self) -> threading.Thread:
+      thread = threading.Thread(target=self._run_send_loop, daemon=True)
+      thread.start()
+      return thread
+
+   def _run_send_loop(self) -> None:
+      asyncio.run(self._ws_send_loop())
+
+   async def _ws_send_loop(self) -> None:
+      async with websockets.connect(self._ws_url) as ws:
+         print(f"Connected to PubSubServer websocket: {self._ws_url}")
+         while True:
+            payload = await asyncio.to_thread(self._outgoing_queue.get)
+            body = json.dumps(payload)
+            print(body)
+            await ws.send(body)

src/DataStreamSuite/DSPublisher/appsettings.Development.json

@@ -0,0 +1,69 @@
+{
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "DisposedConnectionCheckPeriod": 1000,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  },
+  "DSLAPIServer": {
+    "Ip": "localhost",
+    "Port": 19093
+  },
+  "DSPublisher": {
+    "StreamPublishingNodes": [
+      {
+        "Name": "sample_stream",
+        "Expression": "MillisecondStream(1000,5)"
+      }
+    ],
+    "PythonPublishingNodes": [
+      {
+        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
+        "Args": ""
+      }
+    ]
+  }
+}

src/DataStreamSuite/DSPublisher/appsettings.json

@@ -0,0 +1,69 @@
+{
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "DisposedConnectionCheckPeriod": 1000,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  },
+  "DSLAPIServer": {
+    "Ip": "localhost",
+    "Port": 19093
+  },
+  "DSPublisher": {
+    "StreamPublishingNodes": [
+      {
+        "Name": "sample_stream",
+        "Expression": "MillisecondStream(1000,5)"
+      }
+    ],
+    "PythonPublishingNodes": [
+      {
+        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
+        "Args": ""
+      }
+    ]
+  }
+}

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -2,7 +2,8 @@
    using BCG;
 
    [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
+   // example: AddColumnsStream(l,"jc1:001")
+   public class AddColumnsStream : ViewStream {
       public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
          ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/CSVReaderStream.cs

@@ -0,0 +1,18 @@
+namespace bts.BlueOrb {
+   using System;
+
+   using BCG;
+
+   [DataStreamUserRespresentation("CSVReaderStream")]
+   public class CSVReaderStream : ViewStream {
+      public CSVReaderStream(string file_name)
+         : this(file_name, false) { }
+      public CSVReaderStream(string file_name, bool guess_schema) {
+         ViewStream rs = new BCG.CSVReaderStream(file_name);
+         if (guess_schema) {
+            rs = new SchemaGuesserStream(rs);
+         }
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CSVSaverStream.cs

@@ -0,0 +1,13 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("CSVSaverStream")]
+   public class CSVSaverStream: ViewStream {
+      public CSVSaverStream(ViewStream parent_str, string file_name) {
+         BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,9 +1,7 @@
-using System;
-
 namespace BCG {
    [DataStreamUserRespresentation("CountStream")]
    public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
+      IDataPacket? myLastFiredDataPacket;
       private IBlueTableSchema mySchema = new BlueTableSchema("count",
          new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("count", 1, typeof(int)));
@@ -24,7 +22,6 @@ namespace BCG {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
-
       private void myHandleChanges(object newValue) {
          myFireEvent(
             myLastFiredDataPacket =
@@ -39,11 +36,8 @@ namespace BCG {
 
       private IBlueRowCollection myGetRows(object dp, int id) {
          BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+         retVal.addRow(new BlueRow(mySchema, id, ((IDataPacket)dp).newRows.Count));
          return retVal;
       }
-
-
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -15,6 +15,7 @@ namespace bts.BlueOrb {
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_year");
             return rv;
          };
          ViewStream rs = new MapStream(parent_stream,
@@ -30,6 +31,7 @@ namespace bts.BlueOrb {
             rv.setValue($"{inp_col}_hour", dt.Hour);
             rv.setValue($"{inp_col}_minute", dt.Minute);
             rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            rv.setValue($"{inp_col}_year", dt.Date.Year);
             return rv;
          });
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,5 +1,8 @@
 namespace bts.BlueOrb {
    using BCG;
+
+   using Bts.DataStreamCore;
+
    using System;
    using System.Text.RegularExpressions;
 
@@ -21,22 +24,24 @@ namespace bts.BlueOrb {
 
          #region IStreamFilter Members
          public bool filter(IBlueRow dr) {
-            bool rv = true;
             for (int i = 0; i < myCols.Length; i++) {
-               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-               if (!rv)
-                  return rv;
+               var val = dr[myCols[i]];
+               var str = val is DBNull
+                  ? string.Empty
+                  : (val?.ToString() ?? string.Empty);
+               if (!myRegExps[i].IsMatch(str)) {
+                  return false;
+               }
             }
-            return rv;
+            return true;
          }
          #endregion
       }
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
+      BlueTable? myLastFiredRows;
+      IBlueRowCollection? myLastReceivedRows;
       private IStreamFilter myFilter;
       private ViewStream myParentStream;
       private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -52,12 +57,11 @@ namespace bts.BlueOrb {
 
 
       public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         myParentStream = vs;
+         myFilter = filter;
+         myFilterString = filter.ToString() ?? string.Empty;
+         if (filter is IDynamicStreamFilter dynFilter) {
+            dynFilter.onFilterChange += myHandleFilterChange;
          }
       }
 
@@ -86,7 +90,7 @@ namespace bts.BlueOrb {
       }
 
       protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
+         var dp = (IDataPacket)o;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -94,23 +98,27 @@ namespace bts.BlueOrb {
 
       private void myHandleCallbacks(object newValue) {
          DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
+         IDataPacket arrivedPacket = (IDataPacket)newValue;
          myLastReceivedRows = arrivedPacket.newRows;
          if (myLastFiredRows != null) {
             retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
                new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
+            var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
+            foreach (IBlueRow r in arrivedPacket.deletedRows) {
+               if (myLastFiredRows.containsID(r.Id())) {
                   myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
+                  deletedRows.Value.addRow(r);
                }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
+            }
+            var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
+            foreach (IBlueRow r in arrivedPacket.addedRows) {
                if (myFilter.filter(r)) {
                   myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
+                  addedRows.Value.addRow(r);
                }
+            }
          } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            myLastFiredRows = new BlueTable(arrivedPacket.Schema);
             retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
             foreach (IBlueRow r in arrivedPacket.newRows)
                if (myFilter.filter(r))

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -1,12 +1,13 @@
-using System;
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
 
 namespace bts.BlueOrb {
    [DataStreamUserRespresentation("GroupByStream")]
@@ -14,41 +15,45 @@ namespace bts.BlueOrb {
 
       public interface INonIncGroupByEvaluator {
          // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
+         IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[]? get_group_by_cols();
       }
 
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
+         Func<string[]?> _groupby_cols_func;
          Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
                                           Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
+
+         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable? rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
                List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
                if (out_rows != null && out_rows.Count > 0) {
                   rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
                }
-               rv.addRows(out_rows);
+               if (rv != null)
+                  rv.addRows(out_rows);
             }
             return rv;
          }
-         public string[] get_group_by_cols() {
+
+         public string[]? get_group_by_cols() {
             return _groupby_cols_func();
          }
 
       }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
+      ViewStream? _main_stream;
+      INonIncGroupByEvaluator? _groupby_func;
+      BlueRowHasher? _row_hasher;
+      string[]? _groupby_cols;
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
+      IDisposable? subscription;
 
       public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -57,40 +62,52 @@ namespace bts.BlueOrb {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+      public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
+      : this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
 
       // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
+      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
             // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            if (rv == null)
+               return new List<IBlueRow>();
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
          mySetRealSource(rs);
+
+         string[]? groupby_cols_func() {
+            if (string.IsNullOrEmpty(groupby_cols)) {
+               return null;
+            }
+            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
+         }
       }
       public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
       this(parent_str, String.Empty, standard_group_by_text) {
       }
 
+      protected override void myOnFirstSubscriber() {
+         Log.isnull(subscription);
+         subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
+      }
 
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+      protected override void myOnLastSubscriber() {
+         subscription?.Dispose();
+         subscription = null;
+      }
 
       private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
+         Log.notnull(_groupby_func);
+         var dp = (IDataPacket)nv;
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -104,7 +121,10 @@ namespace bts.BlueOrb {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
+         if (total_state == null)
+            // mayeb returna n empty table ?
+            return;
          _timer.Stop();
          var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;
@@ -113,9 +133,10 @@ namespace bts.BlueOrb {
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         var deletedRows = myLastValue is null
+            ? new BlueTable(total_state.schema)
+            : ((IDataPacket)myLastValue).newRows;
+         myFireEvent(new DataPacket(total_state, total_state, deletedRows));
       }
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/HideColumnsStream.cs

@@ -0,0 +1,126 @@
+namespace BlueOrb.Operators.Core {
+   using BCG;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+
+   [DataStreamUserRespresentation("HideColumnsStream")]
+   public class HideColumnsStream : ViewStream {
+      // Wrapper row to hide the columns
+      public class MyRow : IBlueRow {
+         #region IBlueRow members
+         private IBlueTableSchema _schema;
+         private IBlueRow _row;
+         public MyRow(IBlueTableSchema schema, IBlueRow row) {
+            _schema = schema;
+            _row = row;
+         }
+         public IBlueTableSchema schema { get { return _schema; } }
+         #endregion
+
+         #region ICollection members
+         public int Count { get { return schema.columnNames.Length; } }
+
+         public bool IsSynchronized { get { return false; } }
+
+         public object SyncRoot { get { return this; } }
+
+         public void CopyTo(Array array, int index) {
+            int i = index;
+            foreach (object o in this as IEnumerable)
+               array.SetValue(o, i++);
+         }
+         #endregion
+
+         #region IDictionary members
+         public bool IsFixedSize { get { return true; } }
+
+         public bool IsReadOnly { get { return true; } }
+
+         public object this[object key] {
+            get {
+               if (key is string)
+                  return this[(string)key];
+               return this[(int)key];
+            }
+            set { throw new NotSupportedException("The IBlueRow is readonly"); }
+         }
+
+         public object this[int index] {
+            get {
+               return this[schema.columnInfo[index].colName];
+            }
+         }
+
+         public object this[string col_name] {
+            get {
+               if (_schema.contains_column(col_name))
+                  return _row[col_name];
+               else
+                  throw new ApplicationException($"{col_name} not found");
+            }
+         }
+
+         public ICollection Keys { get { return schema.columnNames; } }
+
+         public ICollection Values {
+            get {
+               object[] retVal = new object[schema.columnNames.Length];
+               CopyTo(retVal, 0);
+               return retVal;
+            }
+         }
+
+         public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+         public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+         public bool Contains(object key) { return schema[(string)key] != null; }
+
+         public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+         IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+         public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+         #endregion
+
+      }
+
+      private readonly ViewStream _parent_str;
+      private readonly HashSet<string> _removed_cols;
+      private BlueTable _state;
+      public HideColumnsStream(ViewStream parent_str, string hide_cols_expr) {
+         _parent_str = parent_str;
+         _removed_cols = new HashSet<string>(hide_cols_expr.Split(',', ' '));
+         if (_removed_cols.Contains("id"))
+            _removed_cols.Remove("id");
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _parent_str.onChange += _handle_parent_str_cb;
+      }
+
+      private void _handle_parent_str_cb(object dp_obj) {
+         // this could be made incremental and hence faster
+         DataPacket dp_in = dp_obj as DataPacket;
+         if (_state == null) {
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            foreach (ColumnInfo ci in dp_in.schema.columnInfo) {
+               if (!_removed_cols.Contains(ci.colName))
+                  cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
+            }
+            _state = new BlueTable(new BlueTableSchema(dp_in.schema.tableName, cis.ToArray()));
+         }
+         _state.clear();
+         foreach (IBlueRow r in dp_in.newRows) {
+            _state.addRow(new MyRow(_state.schema, r));
+         }
+         var dp_out = BlueDataUtility.build_dp_inc(_state,
+            myLastValue == null ? new BlueTable(_state.schema) : (myLastValue as DataPacket).newRows);
+         myFireEvent(dp_out);
+      }
+   }
+
+
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/CartesianProductStream.cs

@@ -0,0 +1,14 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   [DataStreamUserRespresentation("CartesianProductStream")]
+   public class CartesianProductStream : ViewStream {
+      public CartesianProductStream(ViewStream ls, ViewStream rs) {
+         ls = new AddColumnsStream(ls, "ljc: 001");
+         rs = new AddColumnsStream(rs, "rjc: 001");
+         ViewStream res = new SimpleJoinStream(ls, "ljc", rs, "rjc");
+         res = new global::BlueOrb.Operators.Core.HideColumnsStream(res, "ljc, rjc");
+         mySetRealSource(res);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/FullJoinStream.cs

@@ -0,0 +1,15 @@
+namespace BlueOrb.Operators.Core {
+   using System;
+
+   using BCG;
+
+   [DataStreamUserRespresentation("FullJoinStream")]
+   // JoinStream is technically a LeftJoin. FullJoin only results in rows that have bpth left and right side
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new global::bts.BlueOrb.SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,59 +1,71 @@
-namespace bts.BlueOrb {
+namespace bts.BlueOrb
+{
    using BCG;
-   using RowHash = System.Int32;
+
+   using Bts.DataStream;
+
+   using Spark.Core.Diagnostics;
+
    using System;
    using System.Collections.Generic;
+   using System.Diagnostics.CodeAnalysis;
    using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   using System.Collections;
-   using System.Data.Entity.Core.Mapping;
+
+   using RowHash = System.Int32;
 
    [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
+   public class SimpleJoinStream : ViewStream
+   {
+      public class JoinBookkeepingData
+      {
          // initialized in cons
-         private readonly List<string> _join_col_names;
-         private readonly bool _is_left;
+         readonly string[] _join_col_names;
          private readonly BlueRowHasher _hasher;
          // initialized on first udpate         
-         private IBlueTableSchema _schema;
-         private int[] _join_cols;
+         IBlueTableSchema? _schema;
+         int[]? _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-         
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
-            _is_left = is_left;
+
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher)
+         {
+            _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
             _hasher = row_hasher;
          }
 
-         public IBlueTableSchema schema => _schema;
+         [NotNullIfNotNull(nameof(_schema))]
+         public IBlueTableSchema? schema => _schema;
 
          public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
 
-         public List<IBlueRow> get_matching_rows (int hash) {
-            List<IBlueRow> rv = null;
-            _hash_row_map.TryGetValue(hash, out rv);
-            return rv;
-         }
+         [MemberNotNull(nameof(_schema))]
+         public IBlueTableSchema RequireSchema() =>
+            ExceptionAssert.MemberNotNull(_schema);
 
-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) { 
+         public List<IBlueRow>? get_matching_rows(int hash) =>
+            _hash_row_map.GetValueOrDefault(hash);
+
+         private RowHash _get_hash(IBlueRow r)
+         {
+            if (_join_cols == null || _join_cols.Length == 0)
+            {
                return 0;
             }
             return _hasher.get_hash(r, _join_cols);
          }
 
-         public void update_data(DataPacket dp) {
-            if (!ready) {               
+         public void update_data(IDataPacket dp)
+         {
+            if (!ready)
+            {
                _schema = dp.newRows.schema;
-               _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+               _join_cols = _schema.ResolveIndices(_join_col_names);
             }
             _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
+            foreach (IBlueRow r in dp.newRows)
+            {
                int hash = _get_hash(r);
-               List<IBlueRow> hash_rows = null;
-               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
+               if (!_hash_row_map.TryGetValue(hash, out var hash_rows))
+               {
                   hash_rows = new List<IBlueRow>();
                   _hash_row_map.Add(hash, hash_rows);
                }
@@ -61,8 +73,11 @@
             }
          }
 
-         public bool ready {
-            get {
+         [MemberNotNullWhen(true, nameof(_schema))]
+         public bool ready
+         {
+            get
+            {
                return _schema != null;
             }
          }
@@ -71,117 +86,146 @@
 
       private ViewStream _ls, _rs;
       private JoinBookkeepingData _ldata, _rdata;
-      private ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema = null;
+      IBlueTableSchema? _schema;
 
       public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc)
+      { }
 
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc)
+      {
          _ls = ls;
          _rs = rs;
          List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
          List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count ==  rj_col_list.Count);
+         Log.assert(lj_col_list.Count == rj_col_list.Count);
          int n_cols = lj_col_list.Count;
          BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
+         _ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
+         _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
       }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _ls.onChange += _ls_onchange;
          _rs.onChange += _rs_onchange;
       }
 
-      private void _ls_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+      private void _ls_onchange(object dp_obj)
+      {
+         var dp = (IDataPacket)dp_obj;
          _ldata.update_data(dp);
          _form_and_fire();
       }
 
-      private void _rs_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket ;
+      private void _rs_onchange(object dp_obj)
+      {
+         var dp = (IDataPacket)dp_obj;
          _rdata.update_data(dp);
          _form_and_fire();
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _ls.onChange -= _ls_onchange;
          _rs.onChange -= _rs_onchange;
       }
 
-      private void _build_schema() {
+      [MemberNotNull(nameof(_schema))]
+      private void _build_schema()
+      {
          if (_schema != null)
             return;
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
-         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
-            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
+         var lschema = Log.notnull(_ldata.schema);
+         var rschema = Log.notnull(_rdata.schema);
+         var cis = new List<ColumnInfo> {
+            new("id", 0, TypeCode.Int32)
+         };
+         foreach (ColumnInfo ci in lschema.columnInfo)
+         {
+            string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
-            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
+         foreach (ColumnInfo ci in rschema.columnInfo)
+         {
+            string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
+         _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
       }
 
-      private void _form_and_fire() {
-         if (!_ldata.ready||!_rdata.ready)
+      private void _form_and_fire()
+      {
+         if (!_ldata.ready || !_rdata.ready)
             return;
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
-         
-         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.schema.columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         };
-         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
-            object[] rv = new object[_schema.columnNames.Length];            
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         };
+
          // this is where the join happens
-         foreach (var kv in _ldata.rows) {
+         foreach (var kv in _ldata.rows)
+         {
             RowHash hash = kv.Key;
             List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
+            List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
 
-            foreach (IBlueRow lr in l_rows) {                              
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
+            foreach (IBlueRow lr in l_rows)
+            {
+               if (r_rows != null)
+               {
+                  foreach (IBlueRow r in r_rows)
+                  {
                      object[] rowvals = build_row_array(res_rows.Count, lr);
                      copy_row(rowvals, r, false);
                      res_rows.Add(rowvals);
                   }
-               } else {
+               }
+               else
+               {
                   object[] rowvals = build_row_array(res_rows.Count, lr);
                   res_rows.Add(rowvals);
                }
             }
          }
          BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
+         foreach (object[] rval in res_rows)
+         {
             rv.addRow(new BlueRow(_schema, rval));
          }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
          myFireEvent(dp);
-      }
 
+         object[] build_row_array(int id, IBlueRow l_row)
+         {
+            object[] rv = new object[_schema.columnNames.Length];
+            for (int i = 0; i < _schema.columnNames.Length; i++)
+            {
+               rv[i] = DBNull.Value;
+            }
+            rv[0] = id;
+            copy_row(rv, l_row, true);
+            return rv;
+         }
+
+         void copy_row(object?[] rval, IBlueRow r, bool is_left)
+         {
+            int offset = 1;
+            JoinBookkeepingData sdata = _ldata;
+            if (is_left)
+            {
+               offset = 1;
+               sdata = _ldata;
+            }
+            else
+            {
+               sdata = _rdata;
+               offset = 1 + _ldata.RequireSchema().columnInfo.Length;
+            }
+            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++)
+            {
+               rval[offset + i] = r[i];
+            }
+         }
+      }
    }
+
 }

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -45,8 +45,8 @@ namespace bts.BlueOrb {
       }
 
       // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
+      public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count == 0) {
             return null;
          }
 
@@ -73,8 +73,8 @@ namespace bts.BlueOrb {
       }
 
       // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
+      public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
+         if (rows.Count == 0) {
             return null;
          }
          IBlueRow ir = rows.ElementAt(start_idx);
@@ -104,8 +104,8 @@ namespace bts.BlueOrb {
       // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
       // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
 
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
+      public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count == 0) {
             return null;
          }
          IBlueRow ir = rolling_rows.ElementAt(0);
@@ -150,6 +150,13 @@ namespace bts.BlueOrb {
                return col_vals.Last();
             case MultiRowOp.StdDev:
                return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            case MultiRowOp.EdgeDiff:
+               return col_vals.First() - col_vals.Last();
+            case MultiRowOp.EdgeDiffPercentage: {
+                  double first = col_vals.First();
+                  double last = col_vals.Last();
+                  return ((first - last) / last) * 100.0;
+               }
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -1,8 +1,7 @@
 namespace bts.BlueOrb {
    using BCG;
 
-   using System.Collections.Generic;
-   using System.Net;
+   using System;
 
    [DataStreamUserRespresentation("MultiRowRollingMathStream")]
    public class MultiRowRollingMathStream : ViewStream {
@@ -24,11 +23,12 @@
       }
 
       private void _parent_str_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = (IDataPacket)dp_obj;
          var rows = dp.newRows.to_list();
-         BlueTable out_table = null;
+         BlueTable? out_table = null;
          for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
+            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
+               ?? throw new InvalidOperationException("group_by_compute returned null row");
             if (out_table == null)
                out_table = new BlueTable(out_r.schema);
             out_table.addRow(out_r);

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,5 +1,8 @@
 namespace bts.BlueOrb {
    using BCG;
+
+   using Spark.Core.Diagnostics;
+
    using System;
    using System.Collections;
    using System.Collections.Generic;
@@ -54,7 +57,7 @@ namespace bts.BlueOrb {
 
       public bool IsReadOnly { get { return true; } }
 
-      public object this[object key] {
+      public object? this[object key] {
          get {
             if (key is string)
                return this[(string)key];
@@ -63,16 +66,15 @@ namespace bts.BlueOrb {
          set { throw new NotSupportedException("The IBlueRow is readonly"); }
       }
 
-      public object this[int index] {
+      public object? this[int index] {
          get {
             return this[schema.columnInfo[index].colName];
          }
       }
 
-      public object this[string col_name] {
+      public object? this[string col_name] {
          get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
                return _row[mapped_to_col];
             }
             return _row[col_name];
@@ -89,7 +91,8 @@ namespace bts.BlueOrb {
          }
       }
 
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Add(object key, object? value) =>
+         throw new NotSupportedException();
 
       public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
 
@@ -114,7 +117,7 @@ namespace bts.BlueOrb {
 
       private ViewStream _stream;
       private string _project_expression;
-      private ProjectData _project_data = null;
+      ProjectData? _project_data;
       //public ProjectData 
       public NewProjectStream(ViewStream vs, string project_expression) {
          _stream = vs;
@@ -128,9 +131,8 @@ namespace bts.BlueOrb {
          return _project_data.out_schema;
       }
 
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
+      public IBlueRow map(IBlueRow row) =>
+         new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
 
       // this is where most of the logic is, in terms of translating cols
       private static ProjectData _create_project_data(IBlueTableSchema input_schema,

src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs

@@ -37,7 +37,8 @@
          // now this function should read _out_csv_filename into a dp
          // and fire it
          _compute_and_write();
-         ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
+         //ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
+         ViewStream rs = new CSVReaderStream(_out_csv_filename);
          IBlueRowCollection nrows = null;
          rs.onChange += (dp_obj) => {
             nrows = (dp_obj as DataPacket).newRows;

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -1,18 +1,25 @@
 using System;
+using System.Data.Entity.Core.Mapping;
 
 using BCG;
 
 namespace bts.BlueOrb {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
-   // into a time series of prices/qty 
+   // into a time series of prices/qty
+   [DataStreamUserRespresentation("TimeSeriesStream")]
    public class TimeSeriesStream : ViewStream {
-      private BlueTable _table;
+      private CircularBufferTable _table;
       private ViewStream _stream;
       private int _id = 0;
+      private int _size = 0;
 
-      public TimeSeriesStream(ViewStream stream) {
+      public TimeSeriesStream(ViewStream stream)
+         : this(stream, Int32.MaxValue) { }
+
+      public TimeSeriesStream(ViewStream stream, int size) {
          _stream = stream;
+         _size = size;
       }
 
       protected override void myOnFirstSubscriber() {
@@ -22,24 +29,27 @@ namespace bts.BlueOrb {
          _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
-
+      private bool _orig_schema_has_timestamp = false;
       private void _onstream_change(object o) {
          DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
          if (_table == null) {
             IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
-            _table = new BlueTable(schema);
+            if (!schema.contains_column("timestamp")) {
+               _orig_schema_has_timestamp = true;
+               schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            }
+            _table = new CircularBufferTable(schema, _size);
          }
-         BlueTable added_rows = new BlueTable(_table.schema);
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue("timestamp", now);
+            if (!_orig_schema_has_timestamp)
+               ar.setValue("timestamp", now);
             ar.setValue("id", _id++);
-            added_rows.addRow(ar);
+            _table.add_row(ar);
          }
-         _table.addRows(added_rows);
-         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
       }
    }
 

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -5,7 +5,8 @@ namespace bts.BlueOrb {
    using mbts;
    using BCG;
 
-   [DataStreamUserRespresentation("UnionStream")]
+   // [DataStreamUserRespresentation("UnionStream")]
+   [DataStreamUserRespresentation("AppendStream")]
    public class UnionStream : ViewStream {
       public UnionStream(ViewStream left_stream, ViewStream right_stream) {
          ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -5,17 +5,6 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
-    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="IronPython" Version="3.4.2" />
@@ -27,4 +16,27 @@
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+        <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
+        <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+        <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+        <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
+        <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="ArithmeticExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="BaseUtilities" Version="$(SparkVersion)" />
+        <PackageReference Include="BooleanExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.ColumnExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="Bts.ColumnSortExpressions" Version="$(SparkVersion)" />
+        <PackageReference Include="DataStreamExpressions" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
+
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -5,19 +5,19 @@ using BCG;
 using bts.api.core;
 
 using Bts.DataStreamCore;
+using Bts.PubSub;
 
 using Microsoft.Extensions.Logging;
 
 using Spark.Core.Disposables;
+using Spark.Core.Threading;
 
 using System;
-using System.Collections;
-using System.Collections.Generic;
 using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
+   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
    public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
@@ -36,15 +36,15 @@ public static class DataStreamExtensions {
       }
    }
 
-   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-      source.Select(dp => {
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+   // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
+   //    source.Select(dp => {
+   //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+   //       return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
 
-         IBlueRowCollection<T> CastRows(ICollection rows) =>
-            new BlueTableSelectorWrapper<T>(
-               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-      });
+   //       IBlueRowCollection<T> CastRows(ICollection rows) =>
+   //          new BlueTableSelectorWrapper<T>(
+   //             rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
+   //    });
 
    public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
       source.AsTables<IBlueRow, int>(r => r, r => r.Id());
@@ -59,36 +59,78 @@ public static class DataStreamExtensions {
    public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
       this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
       rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
-         var bd = res.to_builder();
+      return source.Scan(TableUpdate<T, TKey>.Empty(keySelector), (res, dp) => {
+         var bd = res.ToBuilder();
          dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
          foreach (IBlueRow r in addedRows) {
-            bd.add_or_update(valueSelector(r));
+            bd.AddOrUpdate(valueSelector(r));
          }
          foreach (IBlueRow r in deletedRows) {
-            bd.remove(rowKeySelector(r));
+            bd.Remove(rowKeySelector(r));
          }
-         return bd.to_table();
+         return bd.ToImmutable();
       });
    }
 
-   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+   public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
       source.Scan(
-         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
+         (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
          (res, table) => {
-            var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
+            var resTable = res.Table;
+            if (resTable is null) {
+               if (table.Index.Count == 0) {
+                  return (null, null);
+               }
+               resTable = new(table.Index.Values.First().schema);
+            }
+            var bd = resTable.ToDataPacketBuilder();
+            foreach (var r in table.RemovedItems) {
                bd.Remove(r.Id());
             }
-            foreach (var r in table.added) {
+            foreach (var r in table.AddedItems) {
                bd.AddOrUpdate(r);
             }
-            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
+            var resDp = bd.ToDataPacket(out var newRows);
             return (newRows, resDp);
          })
-         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
+         .Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0))
+         .Select(t => t.Dp!);
+
+   public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.Scan(
+         (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
+         (res, table) => {
+            var bd = res.Table.ToDataPacketBuilder();
+            foreach (var r in table.RemovedItems) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in table.AddedItems) {
+               bd.AddOrUpdate(r);
+            }
+            var resDp = bd.ToDataPacket(out var newRows);
+            return (newRows, resDp);
+         })
+         .Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)
          .Select(t => t.Dp);
 
+   public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
+      return Observable.Create<IDataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<IDataPacket> o) {
+         var adapter = new BlueTableImmutableAdapter<T>(schema);
+         IDataPacket initDataPacket = new DataPacket(schema);
+         return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
+
+         (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
+            (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
+            var bd = state.Adapter.ToDataPacketBuilder();
+            bd.Update(tableUpdate);
+            var newRows = bd.ToImmutable(out var dp);
+            return (newRows, dp);
+         }
+      }
+   }
+
    public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
       source.AsDataPackets(schema).AsViewStream();
 
@@ -131,4 +173,46 @@ public static class DataStreamExtensions {
       }
       return indices;
    }
+
+   public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
+      Observable.Create<IDataPacket>(o => {
+         var inputGate = new AtomicGate();
+         var outputGate = new AtomicGate();
+         BlueTableImmutable? lastTable = null;
+         BlueTableImmutable.DataPacketBuilder? bd = null;
+         IDataPacket? pendingDpOut = null;
+         return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
+
+         void OnNext(IDataPacket value) {
+            using (inputGate.BeginTransaction()) {
+               lastTable ??= new(value.newRows.schema);
+               bd ??= lastTable.ToDataPacketBuilder();
+               if (pendingDpOut is null) {
+                  // The last outgoing datapacket was just fired. Don't merge this update with the
+                  // previous one (otherwise its adds/removes would be fired twice).
+                  bd.Reset();
+               }
+               foreach (IBlueRow r in value.deletedRows) {
+                  bd.Remove(r.Id());
+               }
+               foreach (IBlueRow r in value.addedRows) {
+                  bd.AddOrUpdate(r);
+               }
+               pendingDpOut = bd.ToDataPacket(out lastTable);
+            }
+         }
+
+         void DoWork() {
+            IDataPacket? dpOut = null;
+            if (outputGate.TryEnter()) {
+               using var exit = Disposable.Create(outputGate.Exit);
+               (dpOut, pendingDpOut) = (pendingDpOut, dpOut);
+               if (dpOut is not null) {
+                  o.OnNext(dpOut);
+               }
+            }
+         }
+      });
+
+
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -1,8 +1,9 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using System;
    using BCG;
    using System.Collections.Generic;
    using Bts.DataStreamExpressions;
+   using System.Xml;
 
    // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
    // So for example, if an expresison is passed 
@@ -11,9 +12,10 @@ namespace bts.SilverPine {
       private readonly string _dsl;
       public DSLStream(string dsl) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         // Remove all colons
+         string sanitizedDsl = dsl.Replace(":", "COLON");
          _dsl = dsl;
-         ViewStream rs = build_stream(dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
          mySetRealSource(rs);
       }
 
@@ -24,9 +26,10 @@ namespace bts.SilverPine {
             return rv;
 
          } catch (Exception ex) {
+            //send to blueorb
             Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,14 +1,13 @@
 namespace Bts.DataStream;
 
 using BCG;
-using BCG.Extensions;
 using BCG.Utilities.General;
 
 using bts;
-using bts.collections;
 
 using Bts.DataStreamCore;
 
+using Spark.Core.Collections;
 using Spark.Core.Disposables;
 
 using System;

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -8,13 +8,13 @@ using Microsoft.Extensions.Logging;
 
 using System;
 
-public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
-   readonly IObservable<DataPacket> source;
+public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDisposable {
+   readonly IObservable<IDataPacket> source;
    readonly string? name;
    readonly ILogger? logger;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
+   public ObservableViewStream(IObservable<IDataPacket> source, string? name = null, ILogger? logger = null) {
       this.source = source;
       this.name = name;
       this.logger = logger;
@@ -45,6 +45,6 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
       subscription = null;
    }
 
-   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
+   IDisposable IObservable<IDataPacket>.Subscribe(IObserver<IDataPacket> observer) =>
       source.Subscribe(observer);
 }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -1,14 +1,25 @@
+using bts;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+using Bts.PubSub;
+
+using Microsoft.Extensions.Hosting;
+
+using Spark.Core.Disposables;
+
 using System;
 using System.Collections.Generic;
-
-using Bts.DataStreamExpressions;
-using BCG.Utilities.General;
-using bts;
+using System.IO;
+using System.Reactive.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IWorker {
+   public class RemoteQueryManager : IHostedService {
       public class StreamPubUnit {
          public string name;
          public string stream_expression;
@@ -39,23 +50,46 @@ namespace BCG {
       }
       private IDictionary<string, StreamPubUnit>
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
+      readonly IPubTableBroker _broker;
+      readonly IManagedSubscriber _sub;
+      readonly object syncRoot = new();
+      IDisposable? cloudStreamsSub;
+      IDisposable? remoteRequestsSub;
+      bool isDisposed;
 
 
-      public RemoteQueryManager() {
-         _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+         _sub = managedSubscriber;
+         _broker = pubTableBroker;
       }
-      private void _on_str(object obj)
-      {
-         if (obj is not DataPacket dp) {
-            return;
+
+      public Task StartAsync(CancellationToken cancellationToken) {
+         lock (syncRoot) {
+            Log.assert(!isDisposed);
+            Log.isnull(cloudStreamsSub);
+            Log.isnull(remoteRequestsSub);
+            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
+            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
+               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+            });
          }
+         return Task.CompletedTask;
+      }
+
+      public Task StopAsync(CancellationToken cancellationToken) {
+         lock (syncRoot) {
+            if (!isDisposed) {
+               isDisposed = true;
+               Disposable.dispose(ref cloudStreamsSub);
+               Disposable.dispose(ref remoteRequestsSub);
+            }
+         }
+         return Task.CompletedTask;
+      }
+
+      private void _on_str(DataPacket dp)
+      {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
@@ -72,7 +106,10 @@ namespace BCG {
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
          if (persist) {
-            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
+            string tempDir = Path.GetTempPath();
+        Directory.CreateDirectory(tempDir); // make sure it exists
+
+        stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
          StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
@@ -82,9 +119,5 @@ namespace BCG {
       private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
-
-      public void doWork() {
-
-      }
    }
 }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -9,7 +9,7 @@ namespace BCG {
    [DataStreamUserRespresentation("remoteQuery")]
    public class RemoteQueryStream : ViewStream {
       private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = new PublisherCore();
+      private PublisherCore _publisher_core = PublisherCore.Create();
       private ViewStream _source_str;
 
       public RemoteQueryStream(params string[] lines) {

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -2,9 +2,6 @@ namespace bts {
    using System.IO;
    using BCG;
 
-   using System.IO;
-   using System.Threading.Tasks.Dataflow;
-
    public class RemoteRequest {
       public readonly string stream_expression;
       public readonly string published_table_name;
@@ -28,4 +25,4 @@ namespace bts {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,33 +1,28 @@
+using Spark.Core.Collections;
+
 using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace BCG {   
    public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_single_tok(string exp) {
+      public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
          string[] toks = exp.Split(':');
          string col_name = toks[0].Trim();
          string col_value = toks[1].Trim();
          Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return Tuple.Create(col_name, tup.Item1, tup.Item2);
+         return (col_name, tup.Item1, tup.Item2);
       }
 
-      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
-         string[] toks = exp.Split(",");
-         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
-         foreach (string s in toks) {
-            var tup = parse_single_tok(s.Trim());
-            rv.Add(tup);
-         }
-         return rv;
-      }
+      public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
+         exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
 
-      private Tuple<string, Type, object> _new_col;
-      private List<Tuple<string, Type, object>> _new_col_map;
-      private IBlueTableSchema _out_schema;
+      readonly List<(string Name, Type Type, object Value)> _new_col_map;
+      IBlueTableSchema? _out_schema;
 
       public AddColumnsStream(ViewStream vs, string col_def) {
          _new_col_map = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
+         ViewStream rs = new MapStream(vs, _get_schema, _map);
          mySetRealSource(rs);
       }
 
@@ -44,7 +39,7 @@ namespace BCG {
       private IBlueRow _map(IBlueRow r) {
          MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
          foreach (var m in _new_col_map) {
-            rv.setValue(m.Item1, m.Item3);
+            rv.setValue(m.Name, m.Value);
          }
          return rv;
       }

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs

@@ -1,9 +1,10 @@
-using System;
+using mbts;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using mbts;
 
 namespace BCG
 {
@@ -33,7 +34,7 @@ namespace BCG
 
          private void _stream_onChange(object newValue)
          {
-            DataPacket dp = newValue as DataPacket;
+            var dp = (IDataPacket)newValue;
             if (dp != null && dp.newRows.Count > 0)
             {
                _stream.onChange -= _stream_onChange;
@@ -42,8 +43,8 @@ namespace BCG
          }
       }
 
-      public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
-      public delegate string GetColumnPrefix(object[] transposing_col_values);
+      public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
+      public delegate string GetColumnPrefix(object?[] transposing_col_values);
 
       public delegate string GetColumnSuffix(string col_name);
       private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
@@ -51,33 +52,31 @@ namespace BCG
       private IncrementValues _increment_values;
       private GetColumnPrefix _get_column_prefix;
 
-      private IBlueTableSchema _schema;
+      IBlueTableSchema? _schema;
 
       private string[] _group_by_cols;
       private Pair<string, Type>[] _cols_per_transposed;
       private string[] _transposing_cols;
 
-      private int[] _transposing_col_indices;
+      int[]? _transposing_col_indices;
 
-      private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
+      Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
          : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
                 get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
       { }
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
       {
          if (get_column_suffix_del != null)
             _get_column_suffix = get_column_suffix_del;
          if (transposed_col_value_comparer != null)
          {
-            _transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
-               {
-                  return transposed_col_value_comparer(p1.second, p2.second);
-               });
+            _transpsed_col_value_comparer_wrapper = new((p1, p2) =>
+               transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
          }
          _group_by_cols = group_by_cols.Split(' ');
          _transposing_cols = transposed_cols_group.Split(' ');
@@ -106,7 +105,7 @@ namespace BCG
             col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
          foreach (string tc_prefix in tc_prefixes)
             foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
          IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
          if (_schema == null)
             _schema = schema;
@@ -130,19 +129,14 @@ namespace BCG
 
       public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
       {
-         Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
-         Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
+         var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
+         var group_by_cols_value_dict = new Dictionary<int, object?[]>();
          BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
          Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
          string table_name = string.Empty;
          if (grouped_rows.Count > 0)
          {
-            IBlueTableSchema schema = null;
-            foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
-            {
-               schema = kvp.Value[0].schema;
-               break;
-            }
+            var schema = grouped_rows.Values.First()[0].schema;
             table_name = schema.tableName;
             _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
             int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
@@ -155,23 +149,22 @@ namespace BCG
          else
             return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); 
          Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
-         List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
+         var tc_values = new List<Pair<int, object?[]>>();
          foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
          {
             //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values 
-            Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
+            var incrementing_values = new Dictionary<int, object?[]>();
             foreach (IBlueRow r in kvp.Value)
             {
                int hash = hasher.get_hash(r, _transposing_col_indices);
-               object[] values;
-               if (!incrementing_values.TryGetValue(hash, out values))
+               if (!incrementing_values.TryGetValue(hash, out var values))
                {
                   incrementing_values.Add(hash, _get_per_transposing_col_values(r));
                   string prefix = _get_column_prefix(_get_transposing_col_values(r));
                   if (!tc_prefixes.ContainsKey(hash))
                   {
                      tc_prefixes.Add(hash, prefix);
-                     tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
+                     tc_values.Add(new(hash, _get_transposing_col_values(r)));
                   }
                }
                else
@@ -194,18 +187,18 @@ namespace BCG
          _set_schema(prefixes, group_by_col_info, table_name);
          BlueTable result = new BlueTable(_schema);
          int id = 1;
-         foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
+         foreach (var kvp in all_tc_infos_and_values)
          {
             MutableBlueRow mbr = new MutableBlueRow(_schema);
             mbr.setValue("id", id++);
             for (int i = 0; i < _group_by_cols.Length; i++)
                mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
-            foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
+            foreach (var tc_cols in kvp.Value)
             {
                string prefix = tc_prefixes[tc_cols.Key];
                for (int i = 0; i < _cols_per_transposed.Length; i++)
                {
-                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
+                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
                   if(mbr.schema.columnNames.Contains(column))
                      mbr.setValue(column, tc_cols.Value[i]);
                }
@@ -217,28 +210,25 @@ namespace BCG
 
       #endregion
 
-      private object[] _get_transposing_col_values(IBlueRow row)
-      {
-         object[] values = new object[_transposing_cols.Length];
+      object?[] _get_transposing_col_values(IBlueRow row) {
+         var values = new object?[_transposing_cols.Length];
          for (int i = 0; i < _transposing_cols.Length; i++)
             values[i] = row[_transposing_cols[i]];
          return values;
       }
 
-      private object[] _get_per_transposing_col_values(IBlueRow row)
-      {
-         object[] values = new object[_cols_per_transposed.Length];
+      object?[] _get_per_transposing_col_values(IBlueRow row) {
+         var values = new object?[_cols_per_transposed.Length];
          for (int i = 0; i < _cols_per_transposed.Length; i++)
-            values[i] = row[_cols_per_transposed[i].first];
+            values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
          return values;
       }
 
-      private object[] _get_group_by_cols_values(IBlueRow row)
-      {
-         object[] values = new object[_group_by_cols.Length];
+      object?[] _get_group_by_cols_values(IBlueRow row) {
+         var values = new object?[_group_by_cols.Length];
          for (int i = 0; i < values.Length; i++)
             values[i] = row[_group_by_cols[i]];
          return values;
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs

@@ -1,22 +0,0 @@
-// using System;
-// using BCG;
-// using BCG.Utilities;
-
-// namespace BCG
-// {
-//    public class CartesianProductStream : ViewStream
-//    {
-//       public CartesianProductStream(ViewStream v1, ViewStream v2)
-//       {
-//          mySetRealSource(
-//          new JoinStream (new AddColumnsStream (v1,
-//             "hiddenCartesianColumn=1"),
-//             "hiddenCartesianColumn",
-//             "!id hiddenCartesianColumn dateTime:dateBoundary",
-//             new AddColumnsStream (v2,
-//             "hiddenCartesianColumn=1"),
-//             "hiddenCartesianColumn",
-//             "!id hiddenCartesianColumn"));
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -1,22 +1,19 @@
-using System;
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 
 namespace BCG
 {
    public class DerivativeStream : ViewStream 
    {
-
-      private IBlueTableSchema mySchema;
+      IBlueTableSchema? mySchema;
       private string myDerivandColumn;
       private string myOutputColumn;
       private string myKeyColumn;
       private DateTime myLastPacketArrival = DateTime.MinValue;      
       private ViewStream myDataSource;
-      
-      private IDictionary<string, double> myPreviousValues;
-
+      Dictionary<string, double>? myPreviousValues;
 
       public DerivativeStream(ViewStream input, string keyColumn,  string columnToDifferentiate, string outputColName)
       {
@@ -39,9 +36,9 @@ namespace BCG
 
       void myDataSource_onChange(object newValue)
       {
-         DataPacket dp = (DataPacket)newValue;
+         var dp = (IDataPacket)newValue;
          if (myLastPacketArrival == DateTime.MinValue)
-            mySchema = myBuildOutputSchema(dp.schema);
+            mySchema = myBuildOutputSchema(dp.Schema);
          else
          {
             DateTime now = DateTime.Now;
@@ -55,7 +52,7 @@ namespace BCG
             else
             {
                IBlueRowCollection deleted, added;
-               BlueDataUtility.computeDifferences(thisVersion, ((DataPacket)myLastValue).newRows, out deleted, out added);
+               BlueDataUtility.computeDifferences(thisVersion, ((IDataPacket)myLastValue).newRows, out deleted, out added);
                myFireEvent(new DataPacket(thisVersion, added, deleted));  
             }
          }
@@ -64,12 +61,13 @@ namespace BCG
 
       protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
       {
+         ExceptionAssert.NotNull(mySchema, myPreviousValues);
          object[] outValues = new object[mySchema.columnInfo.Length];
          r.Values.CopyTo(outValues, 0);
 
-         object derivand = r[myDerivandColumn];
-         double lastValue = default(double);
-         if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value)
+         var derivand = r[myDerivandColumn];
+         var key = getRowKey(r);
+         if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
             outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
          else
             outValues[mySchema.columnInfo.Length - 1] = 0.0;
@@ -84,16 +82,19 @@ namespace BCG
          cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double));
          return new BlueTableSchema("derviative of " + inSchema.tableName, cols);
       }
-      protected void myStoreNewPacketInfo(DataPacket dp)
-      {
+
+      protected void myStoreNewPacketInfo(IDataPacket dp) {
          myPreviousValues = new Dictionary<string, double>();
          myLastPacketArrival = DateTime.Now;
          foreach (IBlueRow row in dp.newRows)
          {
-            object derivand = row[myDerivandColumn];
+            var derivand = row[myDerivandColumn];
             if (derivand != null && derivand != DBNull.Value)
-               myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand);
+               myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
          }
       }
+
+      string getRowKey(IBlueRow row) =>
+         row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
    }
 }

src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs

@@ -1,7 +1,9 @@
-using System;
-using System.Collections;
+using mbts;
+
+using Spark.Core.Collections;
+
+using System;
 using System.Collections.Generic;
-using mbts;
 
 namespace BCG
 {
@@ -12,9 +14,9 @@ namespace BCG
    {
       ViewStream myLStream, myRStream;
 
-      DataPacket myLastR = null, myLastL = null;
-
-      BlueTable myLastFired = null;
+      DataPacket? myLastR;
+      DataPacket? myLastL;
+      BlueTable? myLastFired;
 
       double myThreshold = .00005;
 
@@ -24,14 +26,14 @@ namespace BCG
       public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : 
          this(null, v1, v2, thresh, ColumnsToMatch) { }
 
-      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
       {
          myThreshold = thresh;
          myLStream = v1;
          myRStream = v2;
          myColumnsToMatch = columnsToMatch;
       }
-      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
          : this(db,v1,v2,.00005,columnsToMatch)
       {
       }
@@ -50,45 +52,22 @@ namespace BCG
          myRStream.onChange -= myRChange;
       }
 
-      struct ListComparer : IEqualityComparer<List<object>>
-      {
-         public bool Equals(List<object> x, List<object> y)
-         {
-            if (x.Count != y.Count)
-               return false;
-
-            for (int i = 0; i < x.Count; ++i)
-               if (!x[i].Equals(y[i]))
-                  return false;
-            return true;
-         }
-
-         public int GetHashCode(List<object> obj)
-         {
-            int ret = 0;
-            for(int i = 0; i < obj.Count; ++i)
-               ret ^= obj[i].GetHashCode();
-            return ret;
-         }
-      }
-
       void myHandleCallbacks()
       {
-         DataPacket packet = null;
-
+         DataPacket? packet;
          if (myLastL == null || myLastR == null)
          {
             packet = myLastL == null ? myLastR : myLastL;
-            if (packet != null)
+            if (packet is not null) {
                myFireEvent(packet);
-
-            myLastFired = new BlueTable(packet.newRows);
+               myLastFired = new BlueTable(packet.newRows);
+            }
             return;
          }
 
          BlueTable table = new BlueTable(myLastL.newRows.schema);
 
-         Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
+         var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
 
          foreach(BlueRow row in myLastL.newRows)
          {
@@ -118,11 +97,7 @@ namespace BCG
                }
                catch (Exception) { }
             }
-
-            Pair<BlueRow, BlueRow> pair;
-            rowPairs.TryGetValue(hash, out pair);
-
-
+            rowPairs.TryGetValue(hash, out var pair);
             if (pair == null)
                pair = new Pair<BlueRow, BlueRow>(null, row);
             else
@@ -138,8 +113,7 @@ namespace BCG
 
             if (pair.first == null || pair.second == null)
             {
-               BlueRow row = pair.first == null ? pair.second : pair.first;
-
+               var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
                Dictionary<string, object> newRow = new Dictionary<string, object>();
                foreach (ColumnInfo info in row.schema.columnInfo)
                {
@@ -211,4 +185,4 @@ namespace BCG
 
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs

@@ -5,7 +5,7 @@ namespace BCG
 {
 	public class FileViewStream : ViewStream
 	{
-      private DataPacket myDp;
+      DataPacket? myDp;
 		public FileViewStream (string dataFileName, 
          string schemaFileName)
          : this (dataFileName, schemaFileName, Int32.MaxValue)

src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs

@@ -1,8 +1,7 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
+
 namespace BCG
 {
     public class DateBasedFilter: IDynamicStreamFilter, IWorker
@@ -26,20 +25,14 @@ namespace BCG
             myReference = myTodaysDate.AddDays(-1 * myNDays);                     
         }       
         
-        public event FilterChangeDelegate onFilterChange;       
+        public event FilterChangeDelegate? onFilterChange;       
         
-        public bool filter(IBlueRow r)
-        {
-            if (DBNull.Value.Equals(r[myDateColumn]))
-            {
-                //return false;
-                return true; //it can be considered as our assumption  whenever datecolumn is db null we return true 
-            }
-            DateTime t = (DateTime)r[myDateColumn];                   
-            bool returnVallue =  t < myReference ? false : true;
-            return returnVallue;
-        }
-              
+        public bool filter(IBlueRow r) =>
+           r[myDateColumn] switch {
+              DBNull or null => true,
+              DateTime t => t >= myReference,
+              _ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
+           };
 
         public void doWork()
         {
@@ -47,7 +40,7 @@ namespace BCG
             {
                 myTodaysDate = DateTime.Today;
                 myReference = myTodaysDate.AddDays(-1 * myNDays);
-                onFilterChange();
+                onFilterChange?.Invoke();
             }         
         }      
     }

src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs

@@ -1,8 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
 
 namespace BCG
 {
@@ -26,7 +24,7 @@ namespace BCG
 
       #region IDynamicStreamFilter Members
 
-      public event FilterChangeDelegate onFilterChange;
+      public event FilterChangeDelegate? onFilterChange;
 
       #endregion
 
@@ -34,7 +32,10 @@ namespace BCG
 
       public bool filter(IBlueRow r)
       {
-         DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date;
+         DateTime rowDate = r[myDateColName] switch {
+            DBNull or null => DateTime.Now.Date,
+            object obj => (DateTime)obj,
+         };
          DateTime today = DateTime.Now.Date;
          return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
       }
@@ -48,7 +49,7 @@ namespace BCG
          if (DateTime.Now.Date != myLastCheckedTime.Date)
          {
             myLastCheckedTime = DateTime.Now;
-            onFilterChange();
+            onFilterChange?.Invoke();
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,15 +1,23 @@
-using Bts.DataStream;
+using bts;
 
-using mbts.config;
+using Bts.DataStream.Primitives.Filter;
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+using Spark.Core.Disposables;
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
 using System.Text.RegularExpressions;
 
 namespace BCG.StreamWorld.Addition {
    [DataStreamUserRespresentation(false)]
    public class FilterStream : ViewStream {
       private ViewStream myBaseStream;
-      private BlueTable myBaseTable, myStateTable;
+      BlueTable? myBaseTable;
+      BlueTable? myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
@@ -21,8 +29,9 @@ namespace BCG.StreamWorld.Addition {
       public FilterStream(ViewStream vs, IStreamFilter filter) {
          myBaseStream = vs;
          myFilter = filter;
-         if (filter is IDynamicStreamFilter)
-            (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         vs.ErrorChanged += OnError;
+         if (filter is IDynamicStreamFilter dfilter)
+            dfilter.onFilterChange += myHandleFilterChange;
       }
 
 
@@ -32,46 +41,69 @@ namespace BCG.StreamWorld.Addition {
 
       protected override void myOnLastSubscriber() {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
-         myBaseTable.clear();
-         myStateTable.clear();
+         myBaseTable?.clear();
+         myStateTable?.clear();
+      }
+
+      bool filterRow(IBlueRow r) {
+         try {
+            return myFilter.filter(r);
+         } catch (Exception ex) {
+            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+         }
+         return false;
       }
 
       private void myHandleChanges(object newValue) {
-         DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null) {
-            bts.Log.assert(myBaseTable == null);
-            myStateTable = new BlueTable(dp.schema);
-            myBaseTable = new BlueTable(dp.schema);
-         }
-         BlueTable dels = new BlueTable(dp.schema),
-                     adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows) {
-            myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"])) {
-               dels.addRow(r);
-               myStateTable.deleteRow(r);
+         try {
+            var dp = (IDataPacket)newValue;
+            var schema = dp.Schema;
+            init(schema);
+            var dels = new BlueTable(schema);
+            var adds = new BlueTable(schema);
+            foreach (IBlueRow r in dp.deletedRows) {
+               myBaseTable.deleteRow(r);
+               if (myStateTable.containsID(r.Id())) {
+                  dels.addRow(r);
+                  myStateTable.deleteRow(r);
+               }
             }
-         }
-         foreach (IBlueRow r in dp.addedRows) {
-            myBaseTable.addRow(r);
-            if (myFilter.filter(r)) {
-               adds.addRow(r);
-               myStateTable.addRow(r);
+            foreach (IBlueRow r in dp.addedRows) {
+               myBaseTable.addRow(r);
+               if (filterRow(r)) {
+                  adds.addRow(r);
+                  myStateTable.addRow(r);
+               }
             }
+            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
+               myFireEvent(new DataPacket(myStateTable, adds, dels));
+         } catch (Exception ex) {
+            OnError(ex);
          }
-         if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
-            myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
       public IStreamFilter filter {
          set {
             myFilter = value;
             if (myBaseTable != null) {
+               ExceptionAssert.NotNull(myStateTable);
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
-               foreach (IBlueRow r in myBaseTable)
-                  if (myFilter.filter(r))
-                     myStateTable.addRow(r);
+               // If the filter throws an exception, we can rethrow it as a StreamFilterException
+               // and include the 'currentRow' (for which the filter failed) to provide context.
+               IBlueRow? currentRow = null;
+               foreach (IBlueRow r in myBaseTable) {
+                  try {
+                     currentRow = r;
+                     if (filterRow(r)) {
+                        myStateTable.addRow(r);
+                     }
+                  } catch (Exception ex) {
+                     Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+                     OnError(new StreamFilterException(StreamId, StreamName, null, ex, currentRow));
+                  }
+               }
                IBlueRowCollection dels, adds;
                BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
                myFireEvent(new DataPacket(myStateTable, adds, dels));
@@ -83,6 +115,17 @@ namespace BCG.StreamWorld.Addition {
       //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
 
       private void myHandleFilterChange() { filter = myFilter; }
+
+      [MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
+      void init(IBlueTableSchema schema) {
+         if (myStateTable is null) {
+            Log.isnull(myBaseTable);
+            myStateTable = new BlueTable(schema);
+            myBaseTable = new BlueTable(schema);
+         } else {
+            Log.notnull(myBaseTable);
+         }
+      }
    }
 }
 
@@ -106,29 +149,24 @@ namespace BCG
       }
 
       #region IStreamFilter Members
-      public bool filter(IBlueRow dr)
-      {
-         bool rv = true;
-         for (int i = 0; i < myCols.Length; i++)
-         {
-            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-            if (!rv)
-               return rv;
-         }
-         return rv;
-      }
+      public bool filter(IBlueRow dr) =>
+         myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
+            DBNull or null => string.Empty,
+            object obj => obj.ToString() ?? string.Empty
+         }));
       #endregion
    }
 
    public class FilterStream : ViewStream
    {
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
-      private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
+      BlueTable? myLastFiredRows;
+      IBlueRowCollection? myLastReceivedRows;
+      IStreamFilter? myFilter;
+      readonly ViewStream? myParentStream;
+      string? myFilterString;
+      readonly StreamWorld.Addition.FilterStream? myFilterStream;
+      readonly bool myUseNewFilter = false;
+      IDisposable? subscription;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -158,9 +196,10 @@ namespace BCG
          {
             myParentStream = vs;
             myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+            myFilterString = filter.ToString() ?? string.Empty;
+            if (filter is IDynamicStreamFilter dfilter) {
+               dfilter.onFilterChange += myHandleFilterChange;
+            }
          }
       }
 
@@ -184,18 +223,20 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber()
-      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() {
+         ExceptionAssert.IsNull(subscription);
+         subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
+      }
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         Disposable.Dispose(ref subscription);
       }
 
       protected override void myFireEvent(object o)
       {
-         DataPacket dp = o as DataPacket;
+         var dp = (IDataPacket)o;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -204,34 +245,39 @@ namespace BCG
       private void myHandleCallbacks(object newValue)
       {
          DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
+         var arrivedPacket = (IDataPacket)newValue;
          myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null)
-         {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"]))
-               {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r))
-               {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
+         try {
+            if (myLastFiredRows != null)
+            {
+               var addedRows = new BlueTable(myLastFiredRows.schema);
+               var deletedRows = new BlueTable(myLastFiredRows.schema);
+               retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
+               foreach (IBlueRow r in arrivedPacket.deletedRows)
+                  if (myLastFiredRows.containsID(r.Id()))
+                  {
+                     myLastFiredRows.deleteRow(r);
+                     deletedRows.addRow(r);
+                  }
+               foreach (IBlueRow r in arrivedPacket.addedRows)
+                  if (filterRow(r))
+                  {
+                     myLastFiredRows.addRow(r);
+                     addedRows.addRow(r);
+                  }
+            }
+            else
+            {
+               myLastFiredRows = new BlueTable(arrivedPacket.Schema);
+               retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+               foreach (IBlueRow r in arrivedPacket.newRows)
+                  if (filterRow(r))
+                     myLastFiredRows.addRow(r);
+            }
+            myFireEvent(retVal);
+         } catch (Exception ex) {
+            OnError(ex);
          }
-         else
-         {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
       }
 
       public IStreamFilter filter
@@ -240,7 +286,7 @@ namespace BCG
          {
             if (myUseNewFilter)
             {
-               myFilterStream.filter = filter;
+               requireNewFilter().filter = filter;
             }
             else
             {
@@ -249,25 +295,29 @@ namespace BCG
                {
                    if (myLastReceivedRows != null)
                    {
+                     try {
                        BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
                        foreach (IBlueRow r in myLastReceivedRows)
-                           if (myFilter.filter(r))
+                           if (filterRow(r))
                                nRows.addRow(r);
                        IBlueRowCollection dRows = new BlueTable(nRows.schema),
                           aRows = new BlueTable(nRows.schema);                      
                        BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
                        myLastFiredRows = nRows;
                        myFireEvent(new DataPacket(nRows, aRows, dRows));
-                   }
+                     } catch (Exception ex) {
+                        OnError(ex);
+                     }
+                  }
                }
             }
          }
          get
          {
             if (myUseNewFilter)
-               return myFilterStream.filter;
+               return requireNewFilter().filter;
             else
-               return myFilter;
+               return requireOldFilter();
          }
       }
 
@@ -280,7 +330,25 @@ namespace BCG
 
       private void myHandleFilterChange()
       {
-         filter = myFilter;
+         filter = requireOldFilter();
       }
+
+      bool filterRow(IBlueRow r) {
+         try {
+            return requireOldFilter().filter(r);
+         } catch (Exception ex) {
+            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+         }
+         return false;
+      }
+
+      [MemberNotNull(nameof(myFilterStream))]
+      StreamWorld.Addition.FilterStream requireNewFilter() =>
+         ExceptionAssert.MemberNotNull(myFilterStream);
+
+      [MemberNotNull(nameof(myFilter))]
+      IStreamFilter requireOldFilter() =>
+         ExceptionAssert.MemberNotNull(myFilter);
    }
 }

src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs

@@ -1,8 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
+using BCG.Utilities.General;
+
+using System;
 
 namespace BCG
 {    
@@ -29,7 +27,7 @@ namespace BCG
             myReference = myCurrentTime.AddSeconds(-1 * myNSecs);                  
         }
         
-        public event FilterChangeDelegate onFilterChange;       
+        public event FilterChangeDelegate? onFilterChange;       
         
         public bool filter(IBlueRow r)
         {
@@ -38,9 +36,8 @@ namespace BCG
                 //it can be considered as our assumption  whenever datecolumn is db null we return true 
                 return true;  //dont use this for volparamsched use true              
             }
-            DateTime t = (DateTime)r[myDateColumn];                   
-            bool returnVallue =  t < myReference ? false : true;
-            return returnVallue;
+            var t = (DateTime)r[myDateColumn]!;
+            return t >= myReference;
         }
               
 
@@ -51,8 +48,7 @@ namespace BCG
                 myCurrentTime = DateTime.Now;
                 myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
                 myLastTimeFilteredApplied = DateTime.Now;              
-                onFilterChange();
-                
+                onFilterChange?.Invoke();
             }           
         }
     }

src/DataStreamSuite/DataStream/Primitives/Filter/StreamFilterException.cs

@@ -0,0 +1,19 @@
+#nullable enable
+namespace Bts.DataStream.Primitives.Filter;
+
+using BCG;
+
+using System;
+
+public class StreamFilterException : Exception {
+   public StreamFilterException(Guid streamId, string streamName, string? message, Exception? innerException, IBlueRow? badRow = null)
+      : base(message, innerException) {
+      StreamId = streamId;
+      StreamName = streamName;
+      BadRow = badRow;
+   }
+
+   public Guid StreamId { get; }
+   public string StreamName { get; }
+   public IBlueRow? BadRow { get; }
+}

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -71,11 +71,10 @@ namespace BCG {
 
       #region IStreamFilter
       public bool filter(IBlueRow row) {
-         DateTime now = DateTime.Now;
-
-         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
-         if (now > row_time) {
-            TimeSpan span = now - row_time;
+         DateTime now_utc = DateTime.UtcNow;
+         DateTime row_time_utc = row.get_datetime(_time_col, DateTime.Now).ToUniversalTime();
+         if (now_utc > row_time_utc) {
+            TimeSpan span = now_utc - row_time_utc;
             if (span >= _ttl) {
                return (false);
             }
@@ -90,7 +89,7 @@ namespace BCG {
 
          if (now > myNextFilter) {
             // fire event
-            onFilterChange();
+            onFilterChange?.Invoke();
 
             myNextFilter = now.Add(_filter_period);
          }

src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs

@@ -1,11 +1,13 @@
-using System;
+using BCG.Utilities.General;
+
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
-using System.Text;
-using BCG.Utilities.General;
 
 using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
-using System.Diagnostics;
 
 namespace BCG
 {
@@ -40,12 +42,12 @@ namespace BCG
       private IComparer<IBlueRow> myRightCompare;
       protected FuzzyRowComparer myFuzzyCompare;
       protected TimeSpan myTimeWindow;
-      protected IBlueTableSchema myLeftSchema;
-      protected IBlueTableSchema myRightSchema;
-      protected IBlueTableSchema myOutputSchema;
+      protected IBlueTableSchema? myLeftSchema;
+      protected IBlueTableSchema? myRightSchema;
+      protected IBlueTableSchema? myOutputSchema;
       private int myID = 0;
 
-      protected TimeWindowTable myStateRows;
+      protected TimeWindowTable? myStateRows;
 
       private DateTime myNextCull;
       const long MinCullIntervalMS = 1000;
@@ -65,7 +67,7 @@ namespace BCG
          this.myFuzzyCompare = fuzzyCompare;
       }
 
-      public virtual IBlueTableSchema OutputSchema
+      public virtual IBlueTableSchema? OutputSchema
       {
          get
          {
@@ -150,7 +152,7 @@ namespace BCG
          int count;
          for (count = 0; count < list.Count; count++)
          {
-            if ((DateTime)list[count][timeColumn] > expiry)
+            if ((DateTime)list[count][timeColumn]! > expiry)
             {
                // Stop when we find the first row that we can keep.  Since this is sorted all others are also kept.
                break;
@@ -172,16 +174,16 @@ namespace BCG
             this.myFitness = fitness;
          }
          public IBlueRow myRightRow { get; private set; }
-         public IBlueRow myLeftRow { get; set; }
+         public IBlueRow? myLeftRow { get; set; }
          public double myFitness { get; private set; }
-         public IBlueRow myMergedRow { get; set; }
+         public IBlueRow? myMergedRow { get; set; }
 
          #region IComparable<RowMatch> Members
 
-         public int CompareTo(RowMatch other)
-         {
-            return myFitness.CompareTo(other.myFitness);
-         }
+         public int CompareTo(RowMatch? other) =>
+            other is null
+               ? 1
+               : myFitness.CompareTo(other.myFitness);
 
          #endregion
       }
@@ -262,12 +264,12 @@ namespace BCG
 
       private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
       {
+         ExceptionAssert.NotNull(myStateRows);
          // We already emitted correction.MergedRow so we need to redact that and 
          // then emit correction.left as an add since it is no longer matched
          foreach (RowMatch correction in corrections.Distinct())
          {
-            Debug.Assert(correction.myMergedRow != null);
-            this.myStateRows.RemoveRow(correction.myMergedRow);
+            myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow));
             adds.addRow(this.MergeRows(correction.myLeftRow, null));
             dels.addRow(correction.myMergedRow);
          }
@@ -285,7 +287,7 @@ namespace BCG
          int r = 0;
          for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
          {
-            if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
+            if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!)
             {
                // Advance Left
                IEnumerable<RowMatch> rm = rightMatches[l];
@@ -332,7 +334,7 @@ namespace BCG
 
       private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
       {
-         IBlueRow merged = null;
+         IBlueRow? merged = null;
          if (rightMatches != null)
          {
             foreach (RowMatch match in rightMatches)
@@ -366,9 +368,9 @@ namespace BCG
       /// Override this function to provide a custom row merge.
       /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
       /// </summary>
-      protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
+      protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
       {
-         Dictionary<string, object> newRow = new Dictionary<string, object>();
+         Dictionary<string, object?> newRow = new();
 
          if (left == null && right == null) throw new ArgumentException("oh come on.");
 
@@ -393,14 +395,15 @@ namespace BCG
             }
          }
          newRow["id"] = this.myID++;
-         newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
-
+         newRow[NEW_TIME_COLUMN] = left is not null
+            ? left[myLeftTimeColumn]
+            : right![myRightTimeColumn];
          return new BlueRow(this.OutputSchema, newRow);
       }
 
       private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
       {
-         List<RowMatch> matches = null;
+         List<RowMatch>? matches = null;
 
          // Search through all rows in list and compare with the fuzzy comparer
          for (int x = 0; x < list.Count; x++)
@@ -421,8 +424,7 @@ namespace BCG
                matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
 
                // Mark the best match for this particular right hand match
-               RowMatch oldMatch;
-               if (this.myBestMatch.TryGetValue(match, out oldMatch))
+               if (this.myBestMatch.TryGetValue(match, out var oldMatch))
                {
                   if (currentMatch.myFitness < oldMatch.myFitness)
                   {
@@ -441,7 +443,7 @@ namespace BCG
             }
          }
 
-         return matches;
+         return matches ?? Enumerable.Empty<RowMatch>();
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -69,14 +69,7 @@ namespace BCG
       protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
       #endregion
 
-      private IBlueRow myProjectRow (IBlueRow r, string colMapExpression)
-      {
-         BlueTable auxTable = new BlueTable (r.schema);
-         auxTable.addRow (r);
-         BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable);
-         ProjectStream projStream = new ProjectStream (staticStream, colMapExpression);
-         return BlueDataUtility.extractRow (projStream.rows, 0);
-      }
-
+      IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
+         new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project"));
 	}
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs

@@ -1,14 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using System.Collections.Generic;
 
 namespace BCG
 {
    public interface INonIncGroupByEvaluator
    {
-      IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-      string[] get_group_by_cols();
+      IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+      string[]? get_group_by_cols();
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,46 +1,51 @@
-using System;
+using bts;
+using bts.SilverPine;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Diagnostics;
+using Spark.Core.Disposables;
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
-using System.Text.Json;
-
-using bts;
-using bts.SilverPine;
 
 namespace BCG {
    //[DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+         Func<string[]?> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>?> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
                                           Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
+         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable? rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               var out_rows = _evaluate_func(g_rows);
                if (out_rows != null && out_rows.Count > 0) {
                   rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+                  rv.addRows(out_rows);
                }
-               rv.addRows(out_rows);
             }
             return rv;
          }
-         public string[] get_group_by_cols() {
+         public string[]? get_group_by_cols() {
             return _groupby_cols_func();
          }
 
       }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
+      readonly ViewStream? _main_stream;
+      readonly INonIncGroupByEvaluator? _groupby_func;
+      BlueRowHasher? _row_hasher;
+      readonly string[]? _groupby_cols;
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
+      IDisposable? subscription;
 
       public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -49,7 +54,7 @@ namespace BCG {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
@@ -58,7 +63,7 @@ namespace BCG {
 
       public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
-         Func<string[]> groupby_cols_func = () => {
+         Func<string[]?> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
@@ -75,12 +80,17 @@ namespace BCG {
       }
 
 
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
+      protected override void myOnFirstSubscriber() {
+         ExceptionAssert.IsNull(subscription);
+         ExceptionAssert.NotNull(_main_stream).Subscribe(_main_stream_onchange);
+      }
 
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+      protected override void myOnLastSubscriber() =>
+         Disposable.Dispose(ref subscription);
 
       private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
+         ExceptionAssert.NotNull(_groupby_func);
+         var dp = (IDataPacket)nv;
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -94,8 +104,12 @@ namespace BCG {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         var total_state = _groupby_func.evaluate(grouped_rows);
          _timer.Stop();
+         if (total_state is null) {
+            Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
+            return;
+         }
          var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;
          if (applyElapsedMs >= 50) {
@@ -104,8 +118,12 @@ namespace BCG {
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
          // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         if (total_state != null) {
+            var deletedRows = myLastValue is null
+               ? new BlueTable(total_state.schema)
+               : ((IDataPacket)myLastValue).newRows;
+            myFireEvent(new DataPacket(total_state, total_state, deletedRows));
+         }
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs

@@ -1,6 +1,6 @@
 using System;
-using BCG.Utilities;
 using System.Collections;
+using System.Collections.Generic;
 
 namespace BCG
 {
@@ -11,12 +11,10 @@ namespace BCG
    /// </summary>
    public class RowDictionaryTable : IBlueRowCollection
    {
-      protected IDictionary myRowCollections = new Hashtable ();
-      protected BlueTable myLinearCollection;
-      protected string[] myCols;
+      protected readonly Dictionary<string, BlueTable> myRowCollections = new();
+      protected readonly BlueTable myLinearCollection;
+      protected readonly string[] myCols;
       protected IBlueTableSchema mySchema;
-      
-      protected RowDictionaryTable ()  {}
 
       public RowDictionaryTable (IBlueRowCollection rows, params string[] cols)
          : this (rows.schema, cols)
@@ -40,7 +38,7 @@ namespace BCG
 
       public virtual void addRow (IBlueRow row)
       {
-         BlueTable rows = (BlueTable)getGroup (row);
+         var rows = getMutableGroup(row);
          if (rows == null)
          {
             //i.e we dont have any group for this row => we need to create a new group
@@ -53,7 +51,7 @@ namespace BCG
 
       public virtual void removeRow (IBlueRow row)
       {
-         BlueTable rowGroup = (BlueTable)getGroup (row);
+         var rowGroup = requireMutableGroup(row);
 //         BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
          rowGroup.deleteRow (row);
          myLinearCollection.deleteRow (row);
@@ -65,7 +63,7 @@ namespace BCG
 
       public void removeGroup (IBlueRow representativeRow)
       {
-         IBlueRowCollection rowGroup = getGroup (representativeRow);
+         var rowGroup = requireGroup(representativeRow);
          // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
          foreach (IBlueRow row in rowGroup)
             myLinearCollection.deleteRow (row);
@@ -85,24 +83,31 @@ namespace BCG
       /// <returns></returns>
       //public IBlueRowCollection getGroup (IBlueRow row)  {return getGroup (
 
-      public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));}
+      public IBlueRowCollection? getGroup (IBlueRow r) =>
+         getGroup(myRowToString(r));
 
       public ICollection getGroups ()  {return myRowCollections.Values;}
 
-      private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2)
-      {
-         bool retVal = true;
-         foreach (string col in myCols)
-            retVal &= r1[col].Equals(r2[col]);
-         return retVal;
-      }
+      public IBlueRowCollection? getGroup(string repString) =>
+         getMutableGroup(repString);
 
-      public IBlueRowCollection getGroup (string repString)
-      {
-         IBlueRowCollection retVal = myRowCollections[repString] != null ?
-            (IBlueRowCollection) myRowCollections[repString] : null;
-         return retVal;
-      }
+      BlueTable? getMutableGroup(IBlueRow row) =>
+         getMutableGroup(myRowToString(row));
+
+      BlueTable? getMutableGroup(string group) =>
+         myRowCollections.GetValueOrDefault(group);
+
+      IBlueRowCollection requireGroup(IBlueRow row) =>
+         requireGroup(myRowToString(row));
+
+      IBlueRowCollection requireGroup(string group) =>
+         getGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
+
+      BlueTable requireMutableGroup(IBlueRow row) =>
+         requireMutableGroup(myRowToString(row));
+
+      BlueTable requireMutableGroup(string group) =>
+         getMutableGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
 
       private string myRowToString (IBlueRow r)  {return BlueDataUtility.rowToString (r, myCols);}
 

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG
 {
    [DataStreamUserRespresentation (false)]
@@ -17,6 +15,7 @@ namespace BCG
 
       protected override void myInitFiredPacket ()
       {
+         requireSchema();
          myLastFiredPacket = new DataPacket ();
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -25,6 +24,10 @@ namespace BCG
 
       public override void onTableClear (bool isLeft)
       {
+         if (myLastFiredPacket is null) {
+            return;
+         }
+         requireSchema();
          //incase of natural join, left or right wont matter, the end result zeroes out
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs

@@ -1,5 +1,7 @@
+using Spark.Core.Diagnostics;
+
 using System;
-using System.Collections;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BCG
 {
@@ -13,12 +15,10 @@ namespace BCG
       protected JoinOperand myLeftOperand, myRightOperand;
       protected int myLeftJoinColOrdinal, myRightJoinColOrdinal;
       protected ViewStream myLeftStream, myRightStream;
-      protected JoinTableSchema mySchema;
-      protected DataPacket myLastFiredPacket;
+      protected JoinTableSchema? mySchema;
+      protected DataPacket? myLastFiredPacket;
       protected bool myOnceFired = false;
 
-      public IBlueTableSchema schema {get{return mySchema;}}
-
       public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight)
       {
          myLeftStream = vsLeft;
@@ -27,6 +27,9 @@ namespace BCG
          myRightOperand = new JoinOperand (jcRight, false);
       }
 
+      public IBlueTableSchema schema =>
+         requireSchema();
+
       protected override void myOnFirstSubscriber() 
       {
          myLeftOperand.master = this;
@@ -62,29 +65,32 @@ namespace BCG
          {
             myLeftJoinColOrdinal = myLeftOperand.joinColOridinal;
             myRightJoinColOrdinal = myRightOperand.joinColOridinal;
+            myLeftOperand.RequireInitialized();
+            myRightOperand.RequireInitialized();
             mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol, 
                myRightOperand.operandSchema, myRightOperand.joinCol);
             myInitFiredPacket ();
          }
       }
 
+      [MemberNotNull(nameof(mySchema), nameof(myLastFiredPacket))]
       protected virtual void myInitFiredPacket ()
       {
+         var schema = requireSchema();
          myLastFiredPacket = new DataPacket ();
-         myLastFiredPacket.newRows = new JoinTable (mySchema);
-         myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         myLastFiredPacket.deletedRows = new BlueTable (mySchema);
+         myLastFiredPacket.newRows = new JoinTable (schema);
+         myLastFiredPacket.addedRows = new BlueTable (schema);
+         myLastFiredPacket.deletedRows = new BlueTable (schema);
       }
 
-      public IBlueTableSchema getSchema (bool isLeft)
-      {
-         IBlueTableSchema retVal;
-         retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
-         return retVal;
-      }
+      public IBlueTableSchema? getSchema (bool isLeft) =>
+         isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
 
       public void endSession ()
       {
+         if (myLastFiredPacket is null) {
+            return;
+         }
          if (!myOnceFired)
          {
             myOnceFired = true;
@@ -97,13 +103,17 @@ namespace BCG
 
       protected virtual void myInit ()
       {
-         myLeftOperand.init ();
-         myRightOperand.init ();
+         myLeftOperand.Reset ();
+         myRightOperand.Reset ();
          myLastFiredPacket = null;
       }
 
       public virtual void onTableClear (bool isLeft)
       {
+         if (myLastFiredPacket is null) {
+            return;
+         }
+         requireSchema();
          //incase of natural join, left or right wont matter, the end result zeroes out
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinTable (mySchema);
@@ -112,15 +122,19 @@ namespace BCG
 
       public virtual void deleteRows (IBlueRowCollection rows, bool isLeft)
       {
+         var last = myLastFiredPacket;
+         if (last is null) {
+            return;
+         }
          JoinOperand thisOp, otherOp;
          int joinColOrdinal;
          myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft);
          foreach (IBlueRow row in rows)
          {
             thisOp.deleteRow (row);
-            if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0)
+            if (otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)).Count == 0)
                continue;   //row does not participate in join
-            ((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket);
+            ((JoinTable)last.newRows).deleteRow(isLeft, row, last);
          }
       }
 
@@ -135,10 +149,12 @@ namespace BCG
             otherOp = myLeftOperand;
             joinColOrdinal = myRightJoinColOrdinal;
          }
+         var schema = requireSchema();
+         myLastFiredPacket ??= new DataPacket(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema));
          foreach (IBlueRow row in rows)
          {
             thisOp.addRow (row);
-            RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]);
+            RowList matchedNodes = otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal));
             if (matchedNodes.Count == 0)
                continue;   //this row did not participate in the join
             if (isLeft)
@@ -146,10 +162,22 @@ namespace BCG
                   ((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket);
             else
                foreach (RowNode matchedNode in matchedNodes)
-                  ((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket);
+                  ((JoinTable)myLastFiredPacket.newRows).addRow(ExceptionAssert.NotNull(matchedNode.row), row, myLastFiredPacket);
          }
       }
 
+      [MemberNotNull(nameof(mySchema))]
+      protected JoinTableSchema requireSchema() =>
+         ExceptionAssert.NotNull(mySchema);
+
+      [MemberNotNull(nameof(myLastFiredPacket))]
+      protected DataPacket requireLastFiredPacket() {
+         if (myLastFiredPacket is null) {
+            myInitFiredPacket();
+         }
+         return myLastFiredPacket;
+      }
+
       private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft)
       {
          if (isLeft)
@@ -166,6 +194,9 @@ namespace BCG
          }
       }
 
+      object getJoinKey(IBlueRow row, int joinColOrdinal) =>
+         row[joinColOrdinal] ?? throw new ArgumentException("Row has null join key", nameof(row));
+
       /**************************************TEST METHOD(S)*********************************************/
       /// <summary>
       /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -173,8 +204,7 @@ namespace BCG
       /// </summary>
       /// <param name="leftPacket"></param>
       /// <param name="rightPacket"></param>
-      public void test (DataPacket dp, bool isLeft)
-      {
+      public void test(IDataPacket dp, bool isLeft) {
          JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
          jo.changeHandler (dp);
       }

src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs

@@ -1,5 +1,6 @@
-using System;
-using System.Collections;
+using Spark.Core.Diagnostics;
+
+using System.Collections.Generic;
 
 namespace BCG
 {
@@ -8,7 +9,7 @@ namespace BCG
 	/// </summary>
    internal class JoinDeletionIndex
    {
-      IDictionary myIndex = new Hashtable ();
+      Dictionary<object, RowList> myIndex = new();
       private string myOpIdName ;
       private bool myIsLeft;
 
@@ -28,10 +29,8 @@ namespace BCG
       /// <param name="row">JoinRow to add</param>
       public void add (JoinRow row)
       {
-         object o = row[myOpIdName];
-         if (myIndex[o] == null)
-            myIndex[o] = new RowList ();
-         RowNode nodeAdded = ((RowList)myIndex[o]).add (row);
+         object o = ExceptionAssert.NotNull(row[myOpIdName]);
+         RowNode nodeAdded = myIndex.GetOrAdd(o, _ => new()).add(row);
          if (myIsLeft)
             row.leftNode = nodeAdded;
          else
@@ -43,7 +42,10 @@ namespace BCG
       /// To remove a node, if node does not exist exception is thrown
       /// </summary>
       /// <param name="node"></param>
-      public void remove (RowNode node)   {((RowList)myIndex[node.row[myOpIdName]]).remove (node);}
+      public void remove (RowNode node) {
+         var nodeId = ExceptionAssert.NotNull(node.row?[myOpIdName]);
+         myIndex[nodeId].remove(node);
+      }
 
       public RowList getMatchingRows (object key)  {
          RowList retVal = (RowList)myIndex[key];

src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs

@@ -1,19 +1,18 @@
-using System;
-using BCG.Utilities;
-
 using bts;
 
+using Bts.DataStreamCore;
+
 namespace BCG
 {
 	internal class JoinForeignTable : JoinTable
 	{
 		public JoinForeignTable (JoinTableSchema schema) : base (schema) {}
 
-      public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
-      {
-         JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema);
-         Log.assert(myIdIndex[leftRow["id"]] == null);
-         myIdIndex[leftRow["id"]] = newRow;
+      public override void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
+         var newRow = new JoinRow(leftRow, rightRow, mySchema);
+         int leftId = leftRow.Id();
+         Log.assert(myIdIndex[leftId] == null);
+         myIdIndex[leftId] = newRow;
          ((BlueTable)(dp.addedRows)).addRow (newRow);
          myLeftIndex.add (newRow);
          myRightIndex.add (newRow);

src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs

@@ -1,11 +1,12 @@
-using System;
-using System.Collections;
-using BCG.Utilities;
-
 using bts;
 
 using Bts.ColumnMapExpressions;
 
+using Spark.Core.Diagnostics;
+
+using System;
+using System.Collections;
+
 namespace BCG
 {
 	/// <summary>
@@ -37,7 +38,8 @@ namespace BCG
       private JoinTableSchema myJoinSchema;
       private IBlueTableSchema myLeftSchema, myRightSchema;
       private string myLeftColExpression, myRightColExpression;
-      private string[] myUserRules, myResolvedNames;
+      string[] myUserRules;
+      string[]? myResolvedNames;
       private IDictionary myFinalMap = new Hashtable ();
 
 
@@ -87,16 +89,18 @@ namespace BCG
       /// </summary>
       private void myFormUserRules ()
       {
+         var leftExp = ColumnMapExpression.Parse(myLeftColExpression) ?? throw new InvalidOperationException("Failed to parse left column expression");
+         var rightExp = ColumnMapExpression.Parse(myRightColExpression) ?? throw new InvalidOperationException("Failed to parse right column expression");
          if (DataStreamLoadFileConfig.instance.useStaticParser)
          {
-            myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
-            myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
+            myFormUserRules(true, leftExp);
+            myFormUserRules(false, rightExp);
          }
          else
          {
             // TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
-            myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
-            myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
+            myFormUserRules(true, leftExp);
+            myFormUserRules(false, rightExp);
          }
 
       }
@@ -115,13 +119,7 @@ namespace BCG
                OriginalNameInfo oni = myOriginalColInfo[i];
                if (oni.isLeft == isLeft)
                {
-                  if (userMap[oni.colName] != null)
-                  {
-                     if ((string)userMap[oni.colName] != oni.colName)
-                        myUserRules[i] = (string)userMap[oni.colName];
-                     else
-                        myUserRules[i] = oni.colName;
-                  }
+                  myUserRules[i] = userMap[oni.colName] as string ?? oni.colName;
                }
             }
          else
@@ -131,10 +129,13 @@ namespace BCG
                OriginalNameInfo oni = myOriginalColInfo[i];
                if (oni.isLeft == isLeft)
                {
-                  if (userMap[oni.colName] == null)
+                  var colName = userMap[oni.colName] as string;
+                  // myUserRules[i] = userMap[oni.colName] as string 
+                  if (colName is null) {
                      myUserRules[i] = oni.colName;
-                  else if ((string)userMap[oni.colName] != oni.colName)
-                     myUserRules[i] = (string)userMap[oni.colName];
+                  } else if (colName != oni.colName) {
+                     myUserRules[i] = colName;
+                  }
                }
             }
          }
@@ -165,26 +166,12 @@ namespace BCG
 
       private void myFormFinalMap ()
       {
+         ExceptionAssert.NotNull(myResolvedNames);
          foreach (string resolvedName in myResolvedNames)
             if (resolvedName != null)
                myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)];
       }
 
-
-      /// <summary>
-      /// Utility method to invert a hashtable
-      /// </summary>
-      /// <param name="arg"></param>
-      /// <returns></returns>
-   
-      private IDictionary reciprocateDictionary (IDictionary arg)
-      {
-         IDictionary retVal = new Hashtable ();
-         foreach (DictionaryEntry de in arg)
-            retVal[de.Value] = de.Key;
-         return retVal;
-      }
-
       /// <summary>
       /// Utility method to find all occurances of an obkect in an IList
       /// </summary>
@@ -194,9 +181,11 @@ namespace BCG
       private ArrayList myFindAllInCollection (IList list, object obj)
       {
          ArrayList retVal = new ArrayList ();
-         for (int i = 0; i < list.Count; i++)
-            if (list[i] != null && list[i].Equals (obj))
+         for (int i = 0; i < list.Count; i++) {
+            if (Equals(list[i], obj)) {
                retVal.Add (i);
+            }
+         }
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs

@@ -1,6 +1,7 @@
+using Spark.Core.Diagnostics;
+
 using System;
-using System.Collections;
-using BCG.Utilities;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BCG
 {
@@ -11,34 +12,46 @@ namespace BCG
    public class JoinOperand 
    {
       private string myJoinCol;
-      private DataPacket myLastArrivedPacket;
-      private JoinOperand myOtherOperand;
+      IDataPacket? myLastArrivedPacket;
+      JoinOperand? myOtherOperand;
       private bool myIsLeft;
-      private JoinOperandRowIndex myRowIndex;
-      private InternalJoinStream myMaster;
-      private IBlueTableSchema myOpSchema;
-      
+      JoinOperandRowIndex? myRowIndex;
+      InternalJoinStream? myMaster;
+
       public JoinOperand (string joinCol, bool isLeft)
       {
          myJoinCol = joinCol;
          myIsLeft = isLeft;
       }
 
-      internal void changeHandler (object o)
-      {
-         DataPacket temp = (DataPacket)o;
-         if (myLastArrivedPacket != null)
-            mySubsequentChangeHandler ((DataPacket)o);
-         else
-            myFirstChangeHandler ((DataPacket)o);
+      [MemberNotNullWhen(true, nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
+      bool isInitialized {
+         get {
+            if (myLastArrivedPacket is null) {
+               return false;
+            }
+            ExceptionAssert.NotNull(myMaster, myLastArrivedPacket, operandSchema, myOtherOperand, myRowIndex);
+            return true;
+         }
       }
 
-      private void myFirstChangeHandler (DataPacket arrivedDp)
+      internal void changeHandler (object o)
       {
+         var dp = (IDataPacket)o;
+         if (isInitialized) {
+            mySubsequentChangeHandler (dp);
+         } else {
+            myFirstChangeHandler(dp);
+         }
+      }
+
+      [MemberNotNull(nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
+      bool myFirstChangeHandler(IDataPacket arrivedDp) {
+         ExceptionAssert.NotNull(myMaster);
          myLastArrivedPacket = arrivedDp;
-         myOpSchema = myLastArrivedPacket.newRows.schema;
+         operandSchema = myLastArrivedPacket.newRows.schema;
          myOtherOperand = myMaster.getOtherOperand (this);
-         myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo);
+         myRowIndex = new JoinOperandRowIndex (operandSchema[myJoinCol].colNo, operandSchema["id"].colNo);
          if (myOtherOperand.hasReceivedData)
          {
             //the other operand has already received data, so the join would
@@ -49,14 +62,16 @@ namespace BCG
             myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft);
             myMaster.endSession ();
          }
-         else
+         else {
             myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows, 
-               myOpSchema[myJoinCol].colNo,
-               myOpSchema["id"].colNo);
+               operandSchema[myJoinCol].colNo,
+               operandSchema["id"].colNo);
+         }
+         return true;
       }
 
-      private void mySubsequentChangeHandler (DataPacket arrivedDp)
-      {
+      void mySubsequentChangeHandler (IDataPacket arrivedDp) {
+         RequireInitialized();
          myLastArrivedPacket = arrivedDp;
          if (!myOtherOperand.hasReceivedData)
          {
@@ -85,29 +100,46 @@ namespace BCG
          myMaster.endSession ();
       }
 
-      internal void addRow (IBlueRow row) {myRowIndex.add (row);}
+      internal void addRow (IBlueRow row) =>
+         ExceptionAssert.NotNull(myRowIndex).add(row);
 
-      internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);}
+      internal void deleteRow (IBlueRow row) {
+         RequireInitialized();
+         myRowIndex.remove (row);
+      }
 
-      internal IBlueTableSchema operandSchema  {get{return myOpSchema;}}
+      internal IBlueTableSchema? operandSchema { get; private set; }
 
       internal InternalJoinStream master {set{myMaster = value;}}
 
       internal string joinCol   {get{return myJoinCol;}}
 
-      internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}}
+      internal int joinColOridinal {
+         get {
+            RequireInitialized();
+            return operandSchema[myJoinCol].colNo;
+         }
+      }
   
-      internal RowList getMatchingRows (object key)   {return (RowList) myRowIndex.getMatchingRows (key);}
+      internal RowList getMatchingRows (object key) {
+         RequireInitialized();
+         return myRowIndex.getMatchingRows(key);
+      }
 
       internal bool hasReceivedData   {get{return myLastArrivedPacket != null;}}
 
-      internal void init ()  
-      {
+      internal void Reset() {
          ///should be brought back to the the state before the first callback 
          ///(ie the state after construction)
          myRowIndex = null;
          myLastArrivedPacket = null;
       }
-      
+
+      [MemberNotNull(nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
+      internal void RequireInitialized() {
+         if (!isInitialized) {
+            throw new InvalidOperationException("Internal state not properly initialized");
+         }
+      }
    }
 }