cleanup: Remove unnecessary py_module.cpp files

We used to have separate modules for md and execution. I forgot
to remove these after unifying them to a single module.

.config/dotnet-tools.json

@@ -25,12 +25,6 @@
       "commands": [
         "fake"
       ]
-    },
-    "dotnet-t4": {
-      "version": "3.0.0",
-      "commands": [
-        "t4"
-      ]
     }
   }
 }

.devcontainer/Dockerfile

@@ -55,16 +55,14 @@ RUN \
    clangd-$clang_major
 
 # CMake
-RUN set -eux \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends ca-certificates curl gpg \
-  && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
-    | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
-  && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-    > /etc/apt/sources.list.d/kitware.list \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends cmake \
-  && rm -rf /var/lib/apt/lists/*
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
 
 # Dotnet
 RUN apt-get update && apt-get install -y dotnet8
@@ -82,14 +80,9 @@ RUN mkdir -p /etc/apt/keyrings \
 # Emumba (deps)
 RUN apt-get update && apt-get -y install rapidjson-dev
 # Emumba
-ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
-RUN --mount=type=secret,id=bitbucket_access_token \
-  set -eux \
-    && apt-get update \
-    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
-    && curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
-    && apt-get install -y /tmp/emumba-deribit-connector.deb \
-    && rm -f /tmp/emumba-deribit-connector.deb
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
 
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -97,18 +90,6 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
-# Rakurai
-ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
-RUN --mount=type=secret,id=bitbucket_access_token \
-  set -eux \
-    && apt-get update \
-    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
-    && curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
-    # Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
-    # spdlog headers.
-    && apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
-    && rm -f /tmp/rakuirai.deb
-
 # Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \

.devcontainer/devcontainer.json

@@ -1,11 +1,5 @@
 {
-  "build": {
-    "dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
-    "options": [
-      "--secret",
-      "id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
-    ]
-  },
+  "dockerFile": "Dockerfile",
   "hostRequirements": {
     "memory": "8gb",
     "storage": "10gb"
@@ -28,7 +22,6 @@
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
-        "ms-python.python",
         "ms-vscode.cmake-tools",
         "ms-vscode.cpptools",
         "xaver.clang-format"

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.editorconfig

@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles
 
-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower
 
-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion

.env

@@ -46,11 +46,4 @@ SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
 
-### PubSubServer ###
-PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
-PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-
-### MdPublisher ###
-MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
-MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -11,7 +11,6 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
-!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -55,8 +54,6 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
-/src/**/*appsettings.Local.json
-/src/**/*/Configurations/Local/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/

.vscode/launch.json

@@ -14,49 +14,6 @@
       "type": "coreclr",
       "request": "attach"
     },
-    {
-      "name": "Api Server",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
-      "windows": {
-        "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Api Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -69,28 +26,11 @@
       "windows": {
         "program": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
       "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DataStreamPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -142,30 +82,17 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "DataStreamQueryAnalyzer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -191,61 +118,6 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
-    {
-      "name": "PubSubServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sdk Demo",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
-      "windows": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "linux": {
-        "program": "Bts.Spark.Sdk.Demo",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sdk Demo (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "linux": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
     {
       "name": "Spark",
       "type": "coreclr",
@@ -255,9 +127,6 @@
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -270,9 +139,6 @@
       "windows": {
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -334,7 +200,6 @@
       "program": "",
       "linux": {
         "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-        "preLaunchTask": "Build Sparkd (Debug | Clang)",
       },
       "args": [
         "run"
@@ -384,38 +249,8 @@
         }
       ]
     },
-    {
-      "name": "Sparkd Attach",
-      "type": "cppdbg",
-      "request": "attach",
-      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
   ],
   "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "dataStreamPublisherConfig",
       "type": "pickString",
@@ -428,26 +263,6 @@
       ],
       "default": "MarketDataPublisher",
     },
-    {
-      "id": "dataStreamQueryAnalyzerEnvironment",
-      "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "dataStreamPublisherEnvironment",
-      "type": "pickString",
-      "description": "DataStreamPublisher Configuration",
-      "options": [
-        "Local",
-        "ReplayMarketData",
-      ],
-      "default": "Local",
-    },
     {
       "id": "pubSubServerConfig",
       "type": "pickString",
@@ -458,33 +273,12 @@
       ],
       "default": "PubSubServer",
     },
-    {
-      "id": "sdkDemoEnvironment",
-      "type": "pickString",
-      "description": "Sdk Demo Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "sparkConfigServer",
       "type": "promptString",
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
-    {
-      "id": "sparkEnvironment",
-      "type": "pickString",
-      "description": "Spark Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
     {
       "id": "sparkTest",
       "type": "promptString",

.vscode/python.env

@@ -1 +0,0 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py

.vscode/tasks.json

@@ -172,66 +172,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Build Spark Sdk",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark Sdk (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark Sdk (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Build Sparkd",
       "group": "build",
@@ -411,129 +351,6 @@
         "id": "wrench"
       },
     },
-    {
-      "label": "Run Api Server",
-      "dependsOn": "Build Api Server",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug)",
-      "dependsOn": "Build Api Server (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release)",
-      "dependsOn": "Build Api Server (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
     {
       "label": "Run DataStreamQueryAnalyzer",
       "dependsOn": "Build DataStream",
@@ -541,11 +358,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -560,11 +378,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -579,11 +398,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -602,11 +422,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -624,11 +445,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -646,11 +468,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -668,11 +491,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -690,11 +514,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -711,11 +536,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -733,11 +559,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -754,11 +581,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -773,7 +601,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -789,7 +617,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -805,7 +633,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -820,7 +648,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -845,7 +673,7 @@
         "${input:sparkConfigQuery}",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -926,7 +754,7 @@
           "${input:sparkTest}"
         ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
         }
       },
     },
@@ -1024,17 +852,6 @@
     },
   ],
   "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "buildConfiguration",
       "type": "pickString",
@@ -1045,16 +862,6 @@
       ],
       "default": "Debug",
     },
-    {
-      "id": "dataStreamQueryAnalyzerEnvironment",
-      "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
     {
       "id": "sparkdCmakePreset",
       "type": "pickString",
@@ -1078,30 +885,26 @@
       "default": "sparkd-debug-clang",
     },
     {
-      "id": "dataStreamPublisherEnvironment",
+      "id": "dataStreamPublisherConfig",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "Development",
-        "EchoServer",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "PubSubServer",
+        "SimpleEchoClient",
       ],
-      "default": "Development",
+      "default": "PubSubServer",
     },
     {
       "id": "sparkConfigQuery",

Bts.Spark.Sdk.sln

@@ -1,102 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|Any CPU = Release|Any CPU
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.Build.0 = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.Build.0 = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.Build.0 = Release|x64
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.Build.0 = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-	EndGlobalSection
-EndGlobal

CMakeLists.txt

@@ -21,7 +21,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -40,8 +40,9 @@ include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include
 link_directories(${EXTERNALS_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-if(DEFINED APP_DATA_DIR)
-  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
@@ -116,7 +117,7 @@ else()
   if (BUILD_SPARKD)
     add_subdirectory(src/lib_python_interop)
     add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance)
+    add_subdirectory(src/lib_binance_execution)
     add_subdirectory(src/lib_okx_execution)
     add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)

CMakePresets.json

@@ -8,41 +8,36 @@
 
   "configurePresets": [
     {
-      "name": "spark-base",
-      "hidden": true,
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
-      }
-    },
-
-    {
-      "name": "spark-windows",
-      "inherits": "spark-base",
-      "displayName": "Spark",
+      "name": "spark",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
+      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
-        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
       }
     },
 
@@ -58,6 +53,8 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "OFF"
@@ -80,7 +77,8 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
       }
     },
 
@@ -159,17 +157,22 @@
     },
 
     {
-      "name": "spark-linux",
-      "inherits": "spark-base",
-      "displayName": "Spark",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
+      "name": "spark-api-server-windows",
+      "inherits": "spark",
+      "displayName": "Spark Api Server (Windows)",
+      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
+      }
+    },
+
+    {
+      "name": "spark-api-server-linux",
+      "inherits": "sparkd",
+      "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "platform": "linux",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -178,26 +181,21 @@
           "type": "BOOL",
           "value": "OFF"
         },
-        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_EXPORT_COMPILE_COMMANDS": {
           "type": "BOOL",
           "value": "ON"
         },
-        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
-        "DATABENTO_USE_EXTERNAL_JSON": {
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "EXTERNALS_ROOT": "${sourceDir}/externals/linux"
+        }
       }
     },
 
@@ -217,32 +215,92 @@
       "cacheVariables": {
         "CCAPI_LOG_LEVEL": "Trace"
       }
-    }
-  ],
+    },
 
-  "buildPresets": [
     {
-      "name": "spark-windows",
+      "name": "data-stream-suite-windows",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
-      "hidden": true,
-      "configurePreset": "spark-windows"
+      "inherits": "spark",
+      "displayName": "Data Stream Suite (Windows)",
+      "description": "Data Stream utils ported over from Edge",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
     },
 
     {
-      "name": "spark-windows-debug",
+      "name": "data-stream-suite-linux",
+      "displayName": "Data Stream Suite (Linux)",
+      "description": "Data Stream utils ported over from Edge",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "inherits": "sparkd",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
+    }
+  ],
+
+  "buildPresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true
+    },
+
+    {
+      "name": "spark-debug",
       "displayName": "Spark (Debug)",
-      "inherits": "spark-windows",
+      "inherits": "spark",
+      "configurePreset": "spark",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-windows-release",
+      "name": "spark-release",
       "displayName": "Spark (Release)",
-      "inherits": "spark-windows",
+      "inherits": "spark",
+      "configurePreset": "spark",
       "configuration": "Release"
     },
 
@@ -285,27 +343,52 @@
     },
 
     {
-      "name": "spark-linux",
+      "name": "spark-api-server-windows",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true,
+      "configurePreset": "spark-api-server-windows"
+    },
+
+    {
+      "name": "spark-api-server-windows-debug",
+      "displayName": "Spark Api Server (Windows | Debug)",
+      "inherits": "spark-api-server-windows",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "spark-api-server-windows-release",
+      "displayName": "Spark Api Server (Windows | Release)",
+      "inherits": "spark-api-server-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "spark-api-server-linux",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
       "hidden": true,
-      "configurePreset": "spark-linux"
+      "configurePreset": "spark-api-server-linux"
     },
 
     {
-      "name": "spark-linux-debug",
-      "inherits": "spark-linux",
-      "displayName": "Spark (Linux | Debug)",
+      "name": "spark-api-server-linux-debug",
+      "inherits": "spark-api-server-linux",
+      "displayName": "Spark Api Server (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-linux-release",
-      "inherits": "spark-linux",
-      "displayName": "Spark (Linux | Release)",
+      "name": "spark-api-server-linux-release",
+      "inherits": "spark-api-server-linux",
+      "displayName": "Spark Api Server (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -322,6 +405,34 @@
       "configurePreset": "sparkd-ccapi-diag",
       "displayName": "Debug (ccapi diag)",
       "inherits": "sparkd"
+    },
+
+    {
+      "name": "data-stream-suite-windows-debug",
+      "displayName": "Data Stream Suite (Windows | Debug)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-windows-release",
+      "displayName": "Data Stream Suite (Windows | Release)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "data-stream-suite-linux-debug",
+      "displayName": "Data Stream Suite (Linux | Debug)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-linux-release",
+      "displayName": "Data Stream Suite (Linux | Release)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Release"
     }
   ]
 }

Spark.Api.Server.sln

@@ -1,11 +1,10 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
@@ -43,538 +42,168 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
-		MinSizeRel|Any CPU = MinSizeRel|Any CPU
-		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
-		Release|Any CPU = Release|Any CPU
-		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
-		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
-		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.Build.0 = Debug|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.ActiveCfg = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.Build.0 = Release|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.Build.0 = Debug|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.ActiveCfg = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.Build.0 = Release|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.Build.0 = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.Build.0 = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.Build.0 = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.Build.0 = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.Build.0 = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.Build.0 = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.Build.0 = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.Build.0 = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.Build.0 = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.Build.0 = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.Build.0 = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.Build.0 = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.Build.0 = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.Build.0 = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.Build.0 = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.Build.0 = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.Build.0 = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.Build.0 = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.Build.0 = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.Build.0 = Release|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
@@ -586,17 +215,8 @@ Global
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
 	EndGlobalSection
 EndGlobal

cmake/Protobuf.cmake

@@ -0,0 +1,164 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    else()
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    endif()
+
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+
+endfunction()
+

compose.yaml

@@ -1,7 +1,7 @@
 services:
   sparkd:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -12,20 +12,16 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
           set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
+          . /home/bts/sparkdata/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
         ./sparkd run"
     env_file:
       - .env
@@ -41,8 +37,8 @@ services:
 
   sparkd-symbols:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -53,22 +49,18 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        ls -alh /home/bts/.local/share/bts/sparkd/conf/
-        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
           set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
+          . /home/bts/sparkdata/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
         ./sparkd run"
     env_file:
       - .env
@@ -86,15 +78,15 @@ services:
           memory: "${SPARKD__MEMORY:-4gb}"
 
   sparkd-tail:
-    image: bluetradingsystems/linux_dotnet8
+    image: bluetradingsystems/linux_dotnet7
     volumes:
       - type: volume
         read_only: true
         source: sparkd_data
-        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
+        target: /home/bts/sparkdata/data/logging.sparkd
         volume:
           subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
 
   sparkd-ping-test:
     depends_on:
@@ -104,94 +96,11 @@ services:
     build:
       context: .
       dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet8
+        FROM bluetradingsystems/linux_dotnet7
         USER "root"
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
 
-  pubsub:
-    volumes:
-      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
-        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
-    ports:
-      - 22019:22019
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
-        if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
-          mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/pubsub/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/pubsub/conf/.env.local
-          set +a
-        fi
-        ./PubSubServer"
-    env_file:
-      - .env
-      - ${PUBSUB__CONFIG_DIR}/.env
-
-  md-publisher:
-    volumes:
-      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
-      - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
-    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
-        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
-        if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
-          mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/md_publisher/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/md_publisher/conf/.env.local
-          set +a
-        fi
-      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
-    env_file:
-      - .env
-      - ${MD_PUBLISHER__CONFIG_DIR}/.env
-
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
-
 volumes:
   sparkd_data:
     name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
-  pubsub_data:
-    name: "pubsub_22019"
-  md_publisher_data:
-    name: "md_publisher_22019"
-  historic_md:
-    name: "historic_md"

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -1,38 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet8
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts

requirements.txt

@@ -18,10 +18,10 @@ MarkupSafe==3.0.2
 mccabe==0.6.1
 numpy==2.2.6
 platformdirs==2.3.0
-protobuf==6.33.4
+protobuf==4.21.12
 pybars3==0.9.7
-pybind11==3.0.1
-pybind11-stubgen==2.5.5
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
 pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20

spark_client.sln

@@ -1,15 +1,14 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
 EndProject
@@ -19,411 +18,82 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\clie
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{1A5B8C5D-BADC-457E-819F-0050C44FE4D3}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{2A0FBDE0-D870-4567-85BD-F7793A5862C6}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
-		MinSizeRel|Any CPU = MinSizeRel|Any CPU
-		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
-		Release|Any CPU = Release|Any CPU
-		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
-		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
-		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.Build.0 = Debug|Any CPU
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.ActiveCfg = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.Build.0 = Release|Any CPU
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.Build.0 = Debug|Any CPU
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.Build.0 = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.ActiveCfg = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.Build.0 = Release|Any CPU
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.Build.0 = Debug|Any CPU
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.ActiveCfg = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.Build.0 = Release|Any CPU
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.Build.0 = Debug|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.ActiveCfg = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.Build.0 = Release|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{1A5B8C5D-BADC-457E-819F-0050C44FE4D3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6} = {1A5B8C5D-BADC-457E-819F-0050C44FE4D3}
-	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
 	EndGlobalSection

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionGrammar.y

@@ -8,6 +8,7 @@
    public int Integer;
    public double Real;
    public string Identifier;
+   public DateTime DateTime; 
    public ImmutableList<IArithmeticExpression> Arguments;
    public ImmutableList<Assignment> Assignments;
    public IArithmeticExpression Argument;

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -11,20 +11,10 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ArithmeticExpression">
     <Scanner Include="ArithmeticExpressionTokens.lex" />

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -5,5 +5,9 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
-  
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
 </Project>

src/DataStreamSuite/BlueRegistry/DoubleRegistryHandler.cs

@@ -1,6 +1,4 @@
 using System;
-using System.Runtime.Versioning;
-
 using Microsoft.Win32;
 
 namespace BCG
@@ -12,7 +10,6 @@ namespace BCG
          rk.SetValue (name, value.ToString ());
       }
 
-      [SupportedOSPlatform("windows")]
       public static double getValue (RegistryKey rk, string name)
       {
          object v = rk.GetValue (name);

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,80 +6,23 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var obj1 = value1.GetValue(env);
-      var obj2 = value2.GetValue(env);
-      if (obj1 is null) {
-         if (obj2 is null) {
-            return 0;
-         }
-         return -1;
-      }
-      if (obj2 is null) {
-         return 1;
-      }
-      return compareObjs(obj1, obj2);
-   }
-
-   static int compareObjs(object obj1, object obj2) {
+      var cmp1 = value1.Cast<IComparable>();
+      var cmp2 = value2.Cast<IComparable>();
       try {
-         var typ1 = obj1.GetType();
-         var typ2 = obj2.GetType();
-         bool areSameTypes = typ1 == typ2;
-         if (areSameTypes) {
-            return ((IComparable)obj1).CompareTo(obj2);
-         }
-         if (typ1.IsNumeric()) {
-            return tryCompareNumeric(obj1, obj2);
-         } else if (typ2.IsNumeric()) {
-            return -tryCompareNumeric(obj2, obj1);
-         } else if (typ1.IsEnum) {
-            if (typ2 == typeof(string)) {
-               return -((string)obj2).CompareTo(obj1.ToString());
-            }
-         } else if (typ1 == typeof(string)) {
-            return ((string)obj1).CompareTo(obj2.ToString());
-         } else if (typ1 == typeof(DateTime)) {
-            if (typ2 == typeof(TimeSpan)) {
-               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
-            }
-         } else if (typ1 == typeof(TimeSpan)) {
-            if (typ2 == typeof(DateTime)) {
-               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
-            }
-         }
+         var v1 = cmp1.GetValue(env);
+         var v2 = cmp2.GetValue(env);
+         return v1.GetType().IsNumeric() ?
+            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
-      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
-   }
-
-   static int tryCompareNumeric(object numericObj1, object obj2) {
-      var typ2 = obj2.GetType();
-      if (typ2.IsNumeric()) {
-         return new NumericComparer().Compare(numericObj1, obj2);
-      } else if (typ2.IsEnum) {
-         var undTyp2 = typ2.GetEnumUnderlyingType();
-         if (undTyp2 == typeof(byte)) {
-            return new NumericComparer().Compare(numericObj1, (byte)obj2);
-         }
-         if (undTyp2 == typeof(short)) {
-            return new NumericComparer().Compare(numericObj1, (short)obj2);
-         }
-         if (undTyp2 == typeof(int)) {
-            return new NumericComparer().Compare(numericObj1, (int)obj2);
-         }
-         if (undTyp2 == typeof(long)) {
-            return new NumericComparer().Compare(numericObj1, (long)obj2);
-         }
-      }
-      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpression.cs

@@ -1,11 +1,19 @@
 namespace Bts.BooleanExpressions;
 
+using bts;
+
 class BooleanExpression : IBooleanExpression {
    readonly Func<IDictionary<string, object?>, bool> evaluate;
 
    public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
       this.evaluate = evaluate;
 
-   public bool Evaluate(IDictionary<string, object?> env) =>
-      evaluate(env);
+   public bool Evaluate(IDictionary<string, object?> env) {
+      try {
+         return evaluate(env);
+      } catch (Exception ex) {
+         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
+         throw;
+      }
+   }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -47,6 +47,11 @@ IsOfType                 '/'
 
 %%
 
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {BooleanConstant} {
    yylval.Boolean = yytext.ToLower() switch {
       "true" => true,
@@ -57,11 +62,6 @@ IsOfType                 '/'
    return (int)Token.BOOLEAN_CONSTANT;
 }
 
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {NullConstant} {
    yylval.Null = DBNull.Value;
    return (int)Token.NULL_CONSTANT;

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -11,20 +11,10 @@
     <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
   
   <ItemGroup>
     <Scanner Include="BooleanExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -10,20 +10,11 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnExpression">
     <Scanner Include="ColumnExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -10,20 +10,11 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnSortExpression">
     <Scanner Include="ColumnSortExpressionTokens.lex" />

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -7,21 +7,15 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnMapExpression">
     <Scanner Include="ColumnMapExpressionTokens.lex" />

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
-      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
-         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CSVReaderStream.cs

@@ -1,18 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-
-   using BCG;
-
-   [DataStreamUserRespresentation("CSVReaderStream")]
-   public class CSVReaderStream : ViewStream {
-      public CSVReaderStream(string file_name)
-         : this(file_name, false) { }
-      public CSVReaderStream(string file_name, bool guess_schema) {
-         ViewStream rs = new BCG.CSVReaderStream(file_name);
-         if (guess_schema) {
-            rs = new SchemaGuesserStream(rs);
-         }
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CSVSaverStream.cs

@@ -1,13 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("CSVSaverStream")]
-   public class CSVSaverStream: ViewStream {
-      public CSVSaverStream(ViewStream parent_str, string file_name) {
-         BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,7 +1,9 @@
+using System;
+
 namespace BCG {
-   [DataStreamUserRespresentation("CountStream")]
+   [DataStreamUserRespresentation("count")]
    public class CountStream : ViewStream {
-      IDataPacket? myLastFiredDataPacket;
+      private DataPacket myLastFiredDataPacket;
       private IBlueTableSchema mySchema = new BlueTableSchema("count",
          new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("count", 1, typeof(int)));
@@ -22,6 +24,7 @@ namespace BCG {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
+
       private void myHandleChanges(object newValue) {
          myFireEvent(
             myLastFiredDataPacket =
@@ -36,8 +39,11 @@ namespace BCG {
 
       private IBlueRowCollection myGetRows(object dp, int id) {
          BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow(new BlueRow(mySchema, id, ((IDataPacket)dp).newRows.Count));
+         retVal.addRow
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
          return retVal;
       }
+
+
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -15,7 +15,6 @@ namespace bts.BlueOrb {
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_year");
             return rv;
          };
          ViewStream rs = new MapStream(parent_stream,
@@ -31,7 +30,6 @@ namespace bts.BlueOrb {
             rv.setValue($"{inp_col}_hour", dt.Hour);
             rv.setValue($"{inp_col}_minute", dt.Minute);
             rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            rv.setValue($"{inp_col}_year", dt.Date.Year);
             return rv;
          });
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,47 +1,44 @@
 namespace bts.BlueOrb {
    using BCG;
-
-   using Bts.DataStreamCore;
-
    using System;
    using System.Text.RegularExpressions;
 
-   [DataStreamUserRespresentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      public class RegExpFilter : IStreamFilter {
-         private Regex[] myRegExps;
-         private string[] myCols;
 
-         public RegExpFilter(string col, string regExp)
-            : this(new string[] { col }, new string[] { regExp }) { }
+   public class RegExpFilter : IStreamFilter {
+      private Regex[] myRegExps;
+      private string[] myCols;
 
-         public RegExpFilter(string[] cols, string[] regExps) {
-            myCols = cols;
-            myRegExps = new Regex[regExps.Length];
-            for (int i = 0; i < regExps.Length; i++)
-               myRegExps[i] = new Regex(regExps[i]);
-         }
+      public RegExpFilter(string col, string regExp)
+         : this(new string[] { col }, new string[] { regExp }) { }
 
-         #region IStreamFilter Members
-         public bool filter(IBlueRow dr) {
-            for (int i = 0; i < myCols.Length; i++) {
-               var val = dr[myCols[i]];
-               var str = val is DBNull
-                  ? string.Empty
-                  : (val?.ToString() ?? string.Empty);
-               if (!myRegExps[i].IsMatch(str)) {
-                  return false;
-               }
-            }
-            return true;
-         }
-         #endregion
+      public RegExpFilter(string[] cols, string[] regExps) {
+         myCols = cols;
+         myRegExps = new Regex[regExps.Length];
+         for (int i = 0; i < regExps.Length; i++)
+            myRegExps[i] = new Regex(regExps[i]);
       }
-      BlueTable? myLastFiredRows;
-      IBlueRowCollection? myLastReceivedRows;
+
+      #region IStreamFilter Members
+      public bool filter(IBlueRow dr) {
+         bool rv = true;
+         for (int i = 0; i < myCols.Length; i++) {
+            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+            if (!rv)
+               return rv;
+         }
+         return rv;
+      }
+      #endregion
+   }
+
+   //[DataStreamUserReprsentation("FilterStream")]
+   public class FilterStream : ViewStream {
+      private BlueTable myLastFiredRows;
+      private IBlueRowCollection myLastReceivedRows;
       private IStreamFilter myFilter;
       private ViewStream myParentStream;
       private string myFilterString;
+      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -57,11 +54,12 @@ namespace bts.BlueOrb {
 
 
       public FilterStream(ViewStream vs, IStreamFilter filter) {
-         myParentStream = vs;
-         myFilter = filter;
-         myFilterString = filter.ToString() ?? string.Empty;
-         if (filter is IDynamicStreamFilter dynFilter) {
-            dynFilter.onFilterChange += myHandleFilterChange;
+         {
+            myParentStream = vs;
+            myFilter = filter;
+            myFilterString = filter.ToString();
+            if (filter is IDynamicStreamFilter)
+               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
          }
       }
 
@@ -90,7 +88,7 @@ namespace bts.BlueOrb {
       }
 
       protected override void myFireEvent(object o) {
-         var dp = (IDataPacket)o;
+         DataPacket dp = o as DataPacket;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -98,27 +96,23 @@ namespace bts.BlueOrb {
 
       private void myHandleCallbacks(object newValue) {
          DataPacket retVal;
-         IDataPacket arrivedPacket = (IDataPacket)newValue;
+         DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
          if (myLastFiredRows != null) {
             retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
                new BlueTable(myLastFiredRows.schema));
-            var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
-            foreach (IBlueRow r in arrivedPacket.deletedRows) {
-               if (myLastFiredRows.containsID(r.Id())) {
+            foreach (IBlueRow r in arrivedPacket.deletedRows)
+               if (myLastFiredRows.containsID((int)r["id"])) {
                   myLastFiredRows.deleteRow(r);
-                  deletedRows.Value.addRow(r);
+                  (retVal.deletedRows as BlueTable).addRow(r);
                }
-            }
-            var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
-            foreach (IBlueRow r in arrivedPacket.addedRows) {
+            foreach (IBlueRow r in arrivedPacket.addedRows)
                if (myFilter.filter(r)) {
                   myLastFiredRows.addRow(r);
-                  addedRows.Value.addRow(r);
+                  (retVal.addedRows as BlueTable).addRow(r);
                }
-            }
          } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.Schema);
+            myLastFiredRows = new BlueTable(arrivedPacket.schema);
             retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
             foreach (IBlueRow r in arrivedPacket.newRows)
                if (myFilter.filter(r))

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -1,13 +1,12 @@
-using BCG;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Text.Json;
+
+using bts;
+//using bts.SilverPine;
+using BCG;
 
 namespace bts.BlueOrb {
    [DataStreamUserRespresentation("GroupByStream")]
@@ -15,45 +14,41 @@ namespace bts.BlueOrb {
 
       public interface INonIncGroupByEvaluator {
          // grouped_rows
-         IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[]? get_group_by_cols();
+         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[] get_group_by_cols();
       }
 
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]?> _groupby_cols_func;
+         Func<string[]> _groupby_cols_func;
          Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
                                           Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-
-         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable? rv = null;
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
                List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
                if (out_rows != null && out_rows.Count > 0) {
                   rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
                }
-               if (rv != null)
-                  rv.addRows(out_rows);
+               rv.addRows(out_rows);
             }
             return rv;
          }
-
-         public string[]? get_group_by_cols() {
+         public string[] get_group_by_cols() {
             return _groupby_cols_func();
          }
 
       }
-      ViewStream? _main_stream;
-      INonIncGroupByEvaluator? _groupby_func;
-      BlueRowHasher? _row_hasher;
-      string[]? _groupby_cols;
+      private ViewStream _main_stream;
+      private INonIncGroupByEvaluator _groupby_func;
+      private BlueRowHasher _row_hasher;
+      private string[] _groupby_cols;
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
-      IDisposable? subscription;
 
       public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -62,52 +57,40 @@ namespace bts.BlueOrb {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
+      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
-      : this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
 
       // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
             // this si where the group by aggregation happens
-            IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            if (rv == null)
-               return new List<IBlueRow>();
+            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
          mySetRealSource(rs);
-
-         string[]? groupby_cols_func() {
-            if (string.IsNullOrEmpty(groupby_cols)) {
-               return null;
-            }
-            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
-         }
       }
       public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
       this(parent_str, String.Empty, standard_group_by_text) {
       }
 
-      protected override void myOnFirstSubscriber() {
-         Log.isnull(subscription);
-         subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
-      }
 
-      protected override void myOnLastSubscriber() {
-         subscription?.Dispose();
-         subscription = null;
-      }
+      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
+
+      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
 
       private void _main_stream_onchange(object nv) {
-         Log.notnull(_groupby_func);
-         var dp = (IDataPacket)nv;
+         DataPacket dp = nv as DataPacket;
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -121,10 +104,7 @@ namespace bts.BlueOrb {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
-         if (total_state == null)
-            // mayeb returna n empty table ?
-            return;
+         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
          _timer.Stop();
          var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;
@@ -133,10 +113,9 @@ namespace bts.BlueOrb {
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
-         var deletedRows = myLastValue is null
-            ? new BlueTable(total_state.schema)
-            : ((IDataPacket)myLastValue).newRows;
-         myFireEvent(new DataPacket(total_state, total_state, deletedRows));
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
       }
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/Join/INonIncJoinInlineMapper.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+
+namespace bts.BlueOrb {   
+   public interface INonIncJoinInlineMapper
+   {      
+      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
+
+      IBlueRowCollection map_rows(IBlueRowCollection rows);
+      
+      event MapperChangeDelegate onMapperChange;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinBookkeeper.cs

@@ -0,0 +1,152 @@
+using bts;
+using bts.collections;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;
+using BCG;
+
+namespace bts.BlueOrb {   
+   internal class NonIncJoinBookkeeper : IDisposable {
+      public Action on_new_data;
+      private readonly NonIncJoinSpec _join_spec;
+      private readonly int _join_position;
+      private readonly int _start_index_in_output;
+      private readonly BlueRowHasher _row_hasher;
+      private readonly bool _is_left_most;
+      private readonly bool _is_tag_along;
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+
+      private IBlueTableSchema _schema = null;
+      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
+      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
+      private IBlueRow _solitary_row = null;
+
+      private int[] _join_col_indices = null,
+                     _data_col_indices = null;
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
+         _join_spec = the_join_spec;
+         _join_position = join_position;
+         _start_index_in_output = start_index_in_output;
+         _row_hasher = row_hasher;
+         _is_left_most = _join_position == 0;
+         _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
+         _join_spec.stream.onChange += _on_stream_cb;
+      }
+
+      public void _on_stream_cb(object dp_obj) {
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         var timer = Stopwatch.StartNew();
+         DataPacket dp = (DataPacket)dp_obj;
+         _fill_missing_data(dp);
+         if (_schema == null) {
+            _schema = dp.schema;
+            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
+            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
+         }
+
+         if (_is_left_most) {
+            _row_list.Clear();
+            foreach (IBlueRow r in dp.newRows)
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
+            if (_is_tag_along)
+               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
+            else {
+               _row_map.Clear();
+               foreach (IBlueRow r in dp.newRows)
+                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
+            }
+         }
+         timer.Stop();
+         _n_total_fires++;
+         if (timer.ElapsedMilliseconds >= 50) {
+            _n_expensive_fires++;
+            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         if (on_new_data != null)
+            on_new_data();
+      }
+
+      public bool is_ready { get { return _schema != null; } }
+
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
+            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
+      }
+
+      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
+
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+         IBlueRow this_row = _row_list[row_index].second;
+         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+      }
+
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
+            if (_solitary_row != null)
+               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
+            else
+               copy_dbnull_to_output_row(output_row);
+         } else {
+            IBlueRow this_row = null;
+            if (_row_map.TryGetValue(row_hash, out this_row))
+               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+            else
+               copy_dbnull_to_output_row(output_row);
+         }
+      }
+
+      private void copy_dbnull_to_output_row(object[] output_row) {
+         for (int i = 0; i < _data_col_indices.Length; i++)
+            output_row[i + _start_index_in_output] = DBNull.Value;
+      }
+
+      #region IDisposable
+
+      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinSpec.cs

@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using BCG;
+
+using mbts;
+
+namespace bts.BlueOrb {   
+   // this a collection of spec, theres no joining logic in here
+   public class NonIncJoinSpec {
+      public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
+      public string[] join_col_names, data_col_names, projected_col_names;
+
+      public NonIncJoinSpec() { }
+
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
+
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
+         stream = the_stream;
+         join_col_names = the_join_col_names;
+         data_col_names = the_data_col_names;
+         projected_col_names = the_projected_col_names;
+      }
+
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
+         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
+            string[] visible_cols = r.third.Split(' ');
+            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
+            foreach (string s in visible_cols) {
+               string[] toks = s.Split(':');
+               if (toks.Length == 1) {
+                  data_col_names.Add(toks[0]);
+                  mapped_col_names.Add(toks[0]);
+               } else if (toks.Length == 2) {
+                  data_col_names.Add(toks[0]);
+                  mapped_col_names.Add(toks[1]);
+               }
+            }
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+         }
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinStream.cs

@@ -0,0 +1,165 @@
+using bts;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using BCG;
+
+namespace bts.BlueOrb {   
+   [DataStreamUserRespresentation("JoinStream")]
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
+      private readonly NonIncJoinSpec[] _join_specs;
+      private readonly INonIncJoinInlineMapper _output_mapper;
+      private readonly BlueRowHasher _row_hasher;
+      private readonly TimeSpan _throttle_period;
+      private readonly ViewStreamDatabase _vsd;
+
+      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
+      private IBlueTableSchema _output_schema = null;
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private bool _is_dirty = false;
+
+      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
+                              ViewStream right, string right_join_cols, string right_proj)
+         : this(new Triplet<ViewStream, string, string>[]
+               {
+                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
+                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
+               }) { }
+
+      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
+                              params Triplet<ViewStream, string, string>[] join_specs)
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
+      }
+
+      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+         _join_specs = join_specs;
+         _output_mapper = output_mapper;
+         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
+         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
+         _vsd = vsd;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         int data_col_no = 1;
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
+            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            data_col_no += _join_specs[jp].data_col_names.Length;
+            s.on_new_data += _stream_callback;
+            _book_keeping_data.Add(s);
+         }
+         if (_output_mapper != null)
+            _output_mapper.onMapperChange += _on_mapper_change;
+         _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
+      }
+
+      protected override void myOnLastSubscriber() {
+         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+            d.Dispose();
+         if (_output_mapper != null)
+            _output_mapper.onMapperChange -= _on_mapper_change;
+         _vsd.removeWorker(this);
+      }
+
+      private void _stream_callback() {
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         _is_dirty = true;
+         doWork();
+      }
+
+      private void _on_mapper_change() {
+         _is_dirty = true;
+         doWork();
+      }
+
+      private bool _is_ready() {
+         bool rv = true;
+         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+            rv &= d.is_ready;
+         return rv;
+      }
+
+      private void _construct_schema() {
+         if (_output_schema == null) {
+            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
+            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+               d.constribute_cols_to_schema(cis);
+            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
+         }
+      }
+
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+      private void _form_and_fire() {
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
+            _construct_schema();
+            var timer = Stopwatch.StartNew();
+            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
+            List<object[]> output_rows = new List<object[]>();
+
+            for (int i = 0; i < left_row_list.Count; i++) {
+               object[] or = new object[_output_schema.columnInfo.Length];
+               or[0] = i;
+               output_rows.Add(or);
+            }
+
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
+               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
+               for (int i = 0; i < left_row_list.Count; i++) {
+                  if (ji == 0)
+                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
+                  else
+                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
+               }
+            }
+            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
+            timer.Stop();
+            _n_total_fires++;
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+         }
+      }
+
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
+            IBlueRowCollection output_rows;
+            var timer = Stopwatch.StartNew();
+            output_rows = _output_mapper.map_rows(unmapped_rows);
+            timer.Stop();
+            if (timer.ElapsedMilliseconds >= 50) {
+               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
+            }
+            return output_rows;
+         } else {
+            return unmapped_rows;
+         }
+      }
+
+      public void doWork() {
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
+            _is_dirty = false;
+            _last_fire_time = DateTime.Now;
+            _form_and_fire();
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,204 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using Bts.DataStream;
-
-   using Spark.Core.Diagnostics;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Diagnostics.CodeAnalysis;
-   using System.Linq;
-
-   using RowHash = System.Int32;
-
-   [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
-         // initialized in cons
-         readonly string[] _join_col_names;
-         private readonly BlueRowHasher _hasher;
-         // initialized on first udpate         
-         IBlueTableSchema? _schema;
-         int[]? _join_cols;
-         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
-            _hasher = row_hasher;
-         }
-
-         [NotNullIfNotNull(nameof(_schema))]
-         public IBlueTableSchema? schema => _schema;
-
-         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
-
-         [MemberNotNull(nameof(_schema))]
-         public IBlueTableSchema RequireSchema() =>
-            ExceptionAssert.MemberNotNull(_schema);
-
-         public List<IBlueRow>? get_matching_rows(int hash) =>
-            _hash_row_map.GetValueOrDefault(hash);
-
-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) {
-               return 0;
-            }
-            return _hasher.get_hash(r, _join_cols);
-         }
-
-         public void update_data(IDataPacket dp) {
-            if (!ready) {
-               _schema = dp.newRows.schema;
-               _join_cols = _schema.ResolveIndices(_join_col_names);
-            }
-            _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
-               int hash = _get_hash(r);
-               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
-                  hash_rows = new List<IBlueRow>();
-                  _hash_row_map.Add(hash, hash_rows);
-               }
-               hash_rows.Add(r);
-            }
-         }
-
-         [MemberNotNullWhen(true, nameof(_schema))]
-         public bool ready {
-            get {
-               return _schema != null;
-            }
-         }
-
-      }
-
-      private ViewStream _ls, _rs;
-      private JoinBookkeepingData _ldata, _rdata;
-      IBlueTableSchema? _schema;
-
-      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
-
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
-         _ls = ls;
-         _rs = rs;
-         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count == rj_col_list.Count);
-         int n_cols = lj_col_list.Count;
-         BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _ls.onChange += _ls_onchange;
-         _rs.onChange += _rs_onchange;
-      }
-
-      private void _ls_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         _ldata.update_data(dp);
-         _form_and_fire();
-      }
-
-      private void _rs_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         _rdata.update_data(dp);
-         _form_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         _ls.onChange -= _ls_onchange;
-         _rs.onChange -= _rs_onchange;
-      }
-
-      [MemberNotNull(nameof(_schema))]
-      private void _build_schema() {
-         if (_schema != null)
-            return;
-         var lschema = Log.notnull(_ldata.schema);
-         var rschema = Log.notnull(_rdata.schema);
-         var cis = new List<ColumnInfo> {
-            new("id", 0, TypeCode.Int32)
-         };
-         foreach (ColumnInfo ci in lschema.columnInfo) {
-            string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         foreach (ColumnInfo ci in rschema.columnInfo) {
-            string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
-      }
-
-      private void _form_and_fire() {
-         if (!_ldata.ready || !_rdata.ready)
-            return;
-         _build_schema();
-         List<object[]> res_rows = new List<object[]>();
-
-         // this is where the join happens
-         foreach (var kv in _ldata.rows) {
-            RowHash hash = kv.Key;
-            List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
-
-            foreach (IBlueRow lr in l_rows) {
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
-                     object[] rowvals = build_row_array(res_rows.Count, lr);
-                     copy_row(rowvals, r, false);
-                     res_rows.Add(rowvals);
-                  }
-               } else {
-                  object[] rowvals = build_row_array(res_rows.Count, lr);
-                  res_rows.Add(rowvals);
-               }
-            }
-         }
-         BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
-            rv.addRow(new BlueRow(_schema, rval));
-         }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
-         myFireEvent(dp);
-
-         object[] build_row_array(int id, IBlueRow l_row) {
-            object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         }
-
-         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.RequireSchema().columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         }
-      }
-   }
-
-   [DataStreamUserRespresentation("FullJoinStream")]
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -49,14 +49,14 @@ namespace bts.BlueOrb {
          }
       }
    }
-   internal class MSecondStream : TimeUnitStream {
+   public class MSecondStream : TimeUnitStream {
       public MSecondStream(int intervalInms, int size)
          : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
 
       protected override DateTime myRoundTime(DateTime dt) { return dt; }
    }
 
-   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
       private IBlueTableSchema mySchema = new BlueTableSchema("ms",
                                                                new ColumnInfo("id", 0, typeof(int)),
                                                                new ColumnInfo("time", 1, typeof(string)));

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -20,14 +20,13 @@ namespace bts.BlueOrb {
       Sum,
       EdgeDiff,
       EdgeDiffPercentage,
+      //IsEdgeDiff,
    }
    public class MultiRowOpHelper {
-      // input + op
       public readonly List<Tuple<string, MultiRowOp>> operations;
 
       public static MultiRowOpHelper parse_ops(string text) {
          // text is inp_col: op, inp_col: op, inp_col:op
-         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
          List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
          string[] calcs = text.Split(',');
          foreach (string c in calcs) {
@@ -45,8 +44,8 @@ namespace bts.BlueOrb {
       }
 
       // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count == 0) {
+      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
             return null;
          }
 
@@ -72,40 +71,11 @@ namespace bts.BlueOrb {
          return or;
       }
 
-      // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count == 0) {
-            return null;
-         }
-         IBlueRow ir = rows.ElementAt(start_idx);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         // out_col vs inp_cal + type
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-            or.setValue("id", ir["id"]);
-         }
-         return or;
-      }
-
       // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
       // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
 
-      public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count == 0) {
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
             return null;
          }
          IBlueRow ir = rolling_rows.ElementAt(0);
@@ -137,26 +107,15 @@ namespace bts.BlueOrb {
             case MultiRowOp.Min:
                return col_vals.Min<double>();
             case MultiRowOp.Mean:
-               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
+               return col_vals.Sum() / col_vals.Count();
             case MultiRowOp.Sum:
                return col_vals.Sum();
-            case MultiRowOp.Median:
-               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
-            //case MultiRowOp.Mode:
-            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
             case MultiRowOp.First:
                return col_vals.First();
             case MultiRowOp.Last:
                return col_vals.Last();
             case MultiRowOp.StdDev:
                return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            case MultiRowOp.EdgeDiff:
-               return col_vals.First() - col_vals.Last();
-            case MultiRowOp.EdgeDiffPercentage: {
-                  double first = col_vals.First();
-                  double last = col_vals.Last();
-                  return ((first - last) / last) * 100.0;
-               }
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -1,40 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
-   public class MultiRowRollingMathStream : ViewStream {
-      private ViewStream _sorted_parent_str;
-      private MultiRowOpHelper _op;
-      private int _rolling_window_sz;
-      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
-         _sorted_parent_str = new SortStream(parent_str, sort_exp);
-         _op = MultiRowOpHelper.parse_ops(op_txt);
-         _rolling_window_sz = rolling_window_sz;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _sorted_parent_str.onChange += _parent_str_onchange;
-      }      
-
-      protected override void myOnLastSubscriber() {
-         _sorted_parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         var rows = dp.newRows.to_list();
-         BlueTable? out_table = null;
-         for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
-               ?? throw new InvalidOperationException("group_by_compute returned null row");
-            if (out_table == null)
-               out_table = new BlueTable(out_r.schema);
-            out_table.addRow(out_r);
-         }            
-         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,7 +1,7 @@
 namespace bts.BlueOrb {
    using BCG;
 
-   using Spark.Core.Diagnostics;
+   using ProtoBuf;
 
    using System;
    using System.Collections;
@@ -26,7 +26,7 @@ namespace bts.BlueOrb {
    #endregion
 
 
-#region Row with mappings build in
+#region Row with mapipings build in
    internal class NewProjectRow : IBlueRow {
       #region IBlueRow members
       private ProjectData _project_data;
@@ -57,7 +57,7 @@ namespace bts.BlueOrb {
 
       public bool IsReadOnly { get { return true; } }
 
-      public object? this[object key] {
+      public object this[object key] {
          get {
             if (key is string)
                return this[(string)key];
@@ -66,15 +66,16 @@ namespace bts.BlueOrb {
          set { throw new NotSupportedException("The IBlueRow is readonly"); }
       }
 
-      public object? this[int index] {
+      public object this[int index] {
          get {
             return this[schema.columnInfo[index].colName];
          }
       }
 
-      public object? this[string col_name] {
+      public object this[string col_name] {
          get {
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
+            string mapped_to_col = null;
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
                return _row[mapped_to_col];
             }
             return _row[col_name];
@@ -91,8 +92,7 @@ namespace bts.BlueOrb {
          }
       }
 
-      public void Add(object key, object? value) =>
-         throw new NotSupportedException();
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
 
       public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
 
@@ -117,7 +117,7 @@ namespace bts.BlueOrb {
 
       private ViewStream _stream;
       private string _project_expression;
-      ProjectData? _project_data;
+      private ProjectData _project_data = null;
       //public ProjectData 
       public NewProjectStream(ViewStream vs, string project_expression) {
          _stream = vs;
@@ -131,33 +131,42 @@ namespace bts.BlueOrb {
          return _project_data.out_schema;
       }
 
-      public IBlueRow map(IBlueRow row) =>
-         new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
+      public IBlueRow map(IBlueRow row) {
+         return new NewProjectRow(_project_data, row);
+      }
 
-      // this is where most of the logic is, in terms of translating cols
       private static ProjectData _create_project_data(IBlueTableSchema input_schema,
                                                       string project_expression) {
          project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ', ',');
+         string[] project_toks = project_expression.Split(' ');
          HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
+         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
          // by default every col will show up, so if we have the projection as (a:a1)
          // it will convert a to a1, 
-         // modifications (6/30/2025):
-         // only the columns represented here, either mapped
-         // or not, will show
          // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa b") -> aa b
-         foreach (string o_p_tok in project_toks) {
-            string p_tok = o_p_tok.Trim();
-            if (p_tok.Contains(":")) {
-               // explicit mapping
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b")) -> aa b
+         bool hide_all = false;
+         foreach (string p_tok in project_toks) {
+            if (p_tok.StartsWith("!")) {
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
+            } else if (p_tok.Contains(":")) {
                string[] map_toks = p_tok.Split(":");
                string orig_col = map_toks[0];
                string mapped_col = map_toks[1];
-               orig_to_mapped.Add(orig_col, mapped_col);  
+               mapped_to_orig_map.Add(mapped_col, orig_col);
+               orig_to_mapped.Add(orig_col, mapped_col);
             } else {
-               // this is a col_name explicitely stated
                if (input_schema.columnNames.Contains(p_tok)) {
                   string orig_col = p_tok;
                   string mapped_col = orig_col;
@@ -172,19 +181,27 @@ namespace bts.BlueOrb {
             if (ci.colName == "id") {
                ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
                new_cols.Add(nc);
-               continue;           
+               continue;
+            }
+            if (removed_cols.Contains(ci.colName)) {
+               continue;
             } else if (orig_to_mapped.ContainsKey(ci.colName)) {
                string mapped_name = orig_to_mapped[ci.colName];
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
+            } else {
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               //}
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
          return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
       }
 
-   
-
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs

@@ -1,128 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System.IO;
-   using System.Diagnostics;
-   using System;
-
-   [DataStreamUserRespresentation("PythonDataframeStream")]
-   public class PythonDataframeStream : ViewStream {
-      private ViewStream _parent_str;
-      private readonly string _in_csv_file_name;
-      private readonly string _out_csv_filename;
-      private readonly string _transform_function;
-      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
-         var guid = System.Guid.NewGuid();
-         _in_csv_file_name = $"pdfs_in_{guid}.csv";
-         _out_csv_filename = $"pdfs_out_{guid}.csv";
-         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
-         _transform_function = transform_function;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _parent_str_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         Log.assert(dp_obj != null);
-         // the file _in_csv_file_name is now populated.
-         // call py_func_wrapper_main and pass this file name as well
-         // as the output file name
-         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
-         // transform function, which returns the tranformed df. py_func_wrapper_main then
-         // writes it into the _out_csv_filename file.
-         // now this function should read _out_csv_filename into a dp
-         // and fire it
-         _compute_and_write();
-         //ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
-         ViewStream rs = new CSVReaderStream(_out_csv_filename);
-         IBlueRowCollection nrows = null;
-         rs.onChange += (dp_obj) => {
-            nrows = (dp_obj as DataPacket).newRows;
-         };
-         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      private void _compute_and_write() {
-         File.WriteAllText("user_code.py", _transform_function);
-         // wrapper function
-         string py_func_wrapper_main = @"
-import sys
-from user_code import transform
-import pandas as pd
-
-if __name__ == '__main__':
-   input_file_name = sys.argv[1]
-   output_file_name = sys.argv[2]
-# 1. load input_file_name into df
-# 2. do some test calc
-# 3. write the output of 2 into output_file_name
-   df = pd.read_csv(input_file_name)   
-   df_out = transform(df)
-   df_out.to_csv(output_file_name, index=False)";
-
-
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-      }
-
-      #region Unused test code
-      private void _simple_test() {
-
-         // actual function that needs to be called ////////////////////////
-         string py_real_function = @"
-def test(x, out_path):
-   rv = x * 10
-   with open(out_path, 'w') as f:
-      f.write(str(rv))
-   return rv";
-         // the function is in a string object.
-         File.WriteAllText("user_code.py", py_real_function);
-
-
-         // runner app (top level main function) //////////////////////////////
-         string py_func_wrapper_main = @"
-import sys
-from user_code import test
-
-if __name__ == '__main__':
-   arg = int(sys.argv[1])
-   out_path = sys.argv[2]
-   rv = test(arg, out_path)
-   rv = rv + 12
-   with open(out_path, 'w') as f:to \
-      f.write(str(rv))";
-   
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         string outputPath = "py_out.txt";
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py 4 {outputPath}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-         // Now read the output written by the function
-         string result = File.ReadAllText(outputPath);
-         Console.WriteLine(result); // Should print "6"
-
-      }
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs

@@ -1,17 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SortStream")]
-   public class SortStream: ViewStream {
-      public SortStream (ViewStream parent_stream, string sort_expr) {
-         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TimeBasedRollingFilterStream.cs

@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
-   public class TimeBasedRollingFilterStream : ViewStream {
-      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
-         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -1,17 +1,261 @@
+using System;
+using System.Collections.Generic;
+
+using mbts;
+using BCG;
+
 namespace bts.BlueOrb {
-   using System;
-   using System.Collections.Generic;
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
+         private IBlueRow myRow;
+         private int myID;
 
-   using mbts;
-   using BCG;
+         public MyRow(IBlueRow r, int id) {
+            myRow = r;
+            myID = id;
+         }
 
-   // [DataStreamUserRespresentation("UnionStream")]
-   [DataStreamUserRespresentation("AppendStream")]
-   public class UnionStream : ViewStream {
-      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
-         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
-         mySetRealSource(rs);
+         #region IBlueRow Members
+
+         public IBlueTableSchema schema { get { return myRow.schema; } }
+
+         public object this[int index] {
+            get {
+               if (myRow.schema[index].colName == "id")
+                  return myID;
+               else
+                  return myRow[index];
+            }
+         }
+
+         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
+
+         #endregion
+
+         #region IDictionary Members
+
+         public void Add(object key, object value) { throw new Exception("Row is read only."); }
+
+         public void Clear() { throw new Exception("Row is read only."); }
+
+         public bool Contains(object key) { return myRow.Contains(key); }
+
+         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+         public bool IsFixedSize { get { return true; } }
+
+         public bool IsReadOnly { get { return true; } }
+
+         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
+
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+
+         public System.Collections.ICollection Values {
+            get {
+               object[] vals = new object[myRow.Count];
+               CopyTo(vals, 0);
+               return vals;
+            }
+         }
+
+         public object this[object key] {
+            get { return key is string ? this[key as string] : this[(int)key]; }
+            set { throw new Exception("The method or operation is not implemented."); }
+         }
+
+         #endregion
+
+         #region ICollection Members
+
+         public void CopyTo(Array array, int index) {
+            myRow.CopyTo(array, index);
+            array.SetValue(myID, index + myRow.schema["id"].colNo);
+         }
+
+         public int Count { get { return myRow.Count; } }
+
+         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
+
+         public object SyncRoot { get { return myRow.SyncRoot; } }
+
+         #endregion
+
+         #region IEnumerable Members
+
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+         #endregion
+      }
+
+      private BlueTable myRows;
+      private int myID = 0;
+      /// <summary>
+      /// maps the table no to a dictionary containing the
+      /// local IDs to a row in the final big table
+      /// </summary>
+      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
+
+      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
+
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
+         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
+         myRows.addRow(nr);
+         myIDMap[tableNo].Add((int)r["id"], nr);
+         return nr;
+      }
+
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
+         myRows.deleteRow(dr);
+         myIDMap[tableNo].Remove((int)r["id"]);
+         return dr;
+      }
+
+      public IBlueRowCollection removeTable(int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            return null;
+         else {
+            BlueTable rv = new BlueTable(myRows.schema);
+            foreach (IBlueRow r in myIDMap[tableNo].Values)
+               rv.addRow(r);
+            myRows.deleteRows(rv);
+            myIDMap.Remove(tableNo);
+            return rv;
+         }
+      }
+
+      public void clear() {
+         myRows.clear();
+         myIDMap.Clear();
+      }
+
+      #region IBlueRowCollection Members
+
+      public IBlueRow this[int id] { get { return myRows[id]; } }
+
+      public IBlueTableSchema schema { get { return myRows.schema; } }
+
+      public bool containsID(int id) { return myRows.containsID(id); }
+
+      #endregion
+
+      #region ICollection Members
+
+      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
+
+      public int Count { get { return myRows.Count; } }
+
+      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
+
+      public object SyncRoot { get { return myRows.SyncRoot; } }
+
+      #endregion
+
+      #region IEnumerable Members
+
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+
+      #endregion
+   }
+   /// <summary>
+   /// MutableStream can append multiple streams
+   /// as well as give the user the flexibility
+   /// to append (or remove) more streams at
+   /// runtime
+   /// </summary>
+   }
+namespace bts.BlueOrb { 
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
+      private bool mySubscriptionActive = false;
+      private MutableAppendTable myTable;
+      private int myMovingIndex;
+      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
+      private bool _allign_col_orders = true;
+
+      public MutableAppendStream(params ViewStream[] streams)
+         : this(new List<ViewStream>(streams)) { }
+
+      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
+         : this(allign_col_order, new List<ViewStream>(streams)) { }
+
+      public MutableAppendStream(ICollection<ViewStream> streams)
+         : this(true, streams) { }
+
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+         _allign_col_orders = allign_col_orders;
+         foreach (ViewStream vs in streams) {
+            myMovingIndex++;
+            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
+            if (_allign_col_orders)
+               pj = new ProjectStream(pj, "!");
+            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
+         }
+      }
+
+      public void addStream(ViewStream vs) {
+         myMovingIndex++;
+         ///the change name is done to identify the callbacks, and the project to
+         ///make all the schemas col orders the same
+         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
+         if (_allign_col_orders)
+            pj = new ProjectStream(pj, "!");
+         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
+         if (mySubscriptionActive)
+            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
+      }
+
+      public void removeStream(ViewStream vs) {
+         int index = -1;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
+               index = p.Key;
+               if (mySubscriptionActive) {
+                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+                  if (myTable != null) {
+                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
+                     if (delRows != null)
+                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
+                  }
+               }
+               break;
+            }
+            if (index > 0)
+               myStreamIndex.Remove(index);
+         }
+      }
+
+      protected override void myOnFirstSubscriber() {
+         mySubscriptionActive = true;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
+      }
+
+      protected override void myOnLastSubscriber() {
+         mySubscriptionActive = false;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+         if (myTable != null)
+            myTable.clear();
+      }
+
+      private void myHandleStreamChanges(object newValue) {
+         DataPacket dp = newValue as DataPacket;
+         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
+         BlueTable adds = new BlueTable(myTable.schema),
+                     dels = new BlueTable(myTable.schema);
+         int index = Int32.Parse(dp.schema.tableName);
+         foreach (IBlueRow r in dp.deletedRows) {
+            IBlueRow dr = myTable.removeRow(r, index);
+            dels.addRow(dr);
+         }
+         foreach (IBlueRow r in dp.addedRows) {
+            IBlueRow nr = myTable.addRow(r, index);
+            adds.addRow(nr);
+         }
+         myFireEvent(new DataPacket(myTable, adds, dels));
       }
    }
-
 }

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -5,6 +5,17 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
+    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="IronPython" Version="3.4.2" />
@@ -16,27 +27,4 @@
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-        <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
-        <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-        <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-        <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
-        <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ArithmeticExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="BaseUtilities" Version="$(SparkVersion)" />
-        <PackageReference Include="BooleanExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.ColumnExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.ColumnSortExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="DataStreamExpressions" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
-
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,217 +2,25 @@ namespace Bts.DataStream;
 
 using BCG;
 
-using bts.api.core;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using Microsoft.Extensions.Logging;
-
 using Spark.Core.Disposables;
-using Spark.Core.Threading;
 
 using System;
-using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
 
-   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<IDataPacket> observer) {
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
          source.onChange += OnChange;
          return Disposable.Create(() => source.onChange -= OnChange);
 
          void OnChange(object? newValue) =>
-            observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
-               $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
-
-   // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-   //    source.Select(dp => {
-   //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-   //       return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
-
-   //       IBlueRowCollection<T> CastRows(ICollection rows) =>
-   //          new BlueTableSelectorWrapper<T>(
-   //             rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-   //    });
-
-   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
-      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
-      source.AsTables<T, int>(valueSelector, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
-      source.AsTables<T, int>(valueSelector, keySelector);
-
-   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
-      rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.Empty(keySelector), (res, dp) => {
-         var bd = res.ToBuilder();
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         foreach (IBlueRow r in addedRows) {
-            bd.AddOrUpdate(valueSelector(r));
-         }
-         foreach (IBlueRow r in deletedRows) {
-            bd.Remove(rowKeySelector(r));
-         }
-         return bd.ToImmutable();
-      });
-   }
-
-   public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
-      source.Scan(
-         (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
-         (res, table) => {
-            var resTable = res.Table;
-            if (resTable is null) {
-               if (table.Index.Count == 0) {
-                  return (null, null);
-               }
-               resTable = new(table.Index.Values.First().schema);
-            }
-            var bd = resTable.ToDataPacketBuilder();
-            foreach (var r in table.RemovedItems) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.AddedItems) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacket(out var newRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0))
-         .Select(t => t.Dp!);
-
-   public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.Scan(
-         (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
-         (res, table) => {
-            var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.RemovedItems) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.AddedItems) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacket(out var newRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)
-         .Select(t => t.Dp);
-
-   public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
-      return Observable.Create<IDataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<IDataPacket> o) {
-         var adapter = new BlueTableImmutableAdapter<T>(schema);
-         IDataPacket initDataPacket = new DataPacket(schema);
-         return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
-
-         (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
-            (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
-            var bd = state.Adapter.ToDataPacketBuilder();
-            bd.Update(tableUpdate);
-            var newRows = bd.ToImmutable(out var dp);
-            return (newRows, dp);
-         }
-      }
-   }
-
-   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.AsDataPackets(schema).AsViewStream();
-
-   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
-      var indices = ResolveIndices(schema, new[] { name });
-      return indices[0];
-   }
-
-   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
-      var indices = ResolveIndices(schema, new[] { name1, name2 });
-      return (indices[0], indices[1]);
-   }
-
-   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
-      return (indices[0], indices[1], indices[2]);
-   }
-
-   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
-      return (indices[0], indices[1], indices[2], indices[3]);
-   }
-
-   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
-      var columns = schema.columnInfo;
-      var indices = new int[columnNames.Length];
-      int nResolved = 0;
-      for (int i = 0; i < columnNames.Length; i++) {
-         int origNResolved = nResolved;
-         for (int j = 0; j < columns.Length; j++) {
-            if (columns[j].colName == columnNames[i]) {
-               indices[i] = j;
-               nResolved++;
-               break;
-            }
-         }
-         if (nResolved == origNResolved) {
-            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
-         }
-      }
-      return indices;
-   }
-
-   public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
-      Observable.Create<IDataPacket>(o => {
-         var inputGate = new AtomicGate();
-         var outputGate = new AtomicGate();
-         BlueTableImmutable? lastTable = null;
-         BlueTableImmutable.DataPacketBuilder? bd = null;
-         IDataPacket? pendingDpOut = null;
-         return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
-
-         void OnNext(IDataPacket value) {
-            using (inputGate.BeginTransaction()) {
-               lastTable ??= new(value.newRows.schema);
-               bd ??= lastTable.ToDataPacketBuilder();
-               if (pendingDpOut is null) {
-                  // The last outgoing datapacket was just fired. Don't merge this update with the
-                  // previous one (otherwise its adds/removes would be fired twice).
-                  bd.Reset();
-               }
-               foreach (IBlueRow r in value.deletedRows) {
-                  bd.Remove(r.Id());
-               }
-               foreach (IBlueRow r in value.addedRows) {
-                  bd.AddOrUpdate(r);
-               }
-               pendingDpOut = bd.ToDataPacket(out lastTable);
-            }
-         }
-
-         void DoWork() {
-            IDataPacket? dpOut = null;
-            if (outputGate.TryEnter()) {
-               using var exit = Disposable.Create(outputGate.Exit);
-               (dpOut, pendingDpOut) = (pendingDpOut, dpOut);
-               if (dpOut is not null) {
-                  o.OnNext(dpOut);
-               }
-            }
-         }
-      });
-
-
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -1,9 +1,8 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using System;
    using BCG;
    using System.Collections.Generic;
    using Bts.DataStreamExpressions;
-   using System.Xml;
 
    // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
    // So for example, if an expresison is passed 
@@ -12,10 +11,9 @@ namespace bts.BlueOrb {
       private readonly string _dsl;
       public DSLStream(string dsl) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
-         // Remove all colons
-         string sanitizedDsl = dsl.Replace(":", "COLON");
          _dsl = dsl;
-         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         ViewStream rs = build_stream(dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
          mySetRealSource(rs);
       }
 
@@ -26,10 +24,9 @@ namespace bts.BlueOrb {
             return rv;
 
          } catch (Exception ex) {
-            //send to blueorb
             Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,13 +1,14 @@
 namespace Bts.DataStream;
 
 using BCG;
+using BCG.Extensions;
 using BCG.Utilities.General;
 
 using bts;
+using bts.collections;
 
 using Bts.DataStreamCore;
 
-using Spark.Core.Collections;
 using Spark.Core.Disposables;
 
 using System;

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -4,21 +4,14 @@ using BCG;
 
 using bts;
 
-using Microsoft.Extensions.Logging;
-
 using System;
 
-public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDisposable {
-   readonly IObservable<IDataPacket> source;
-   readonly string? name;
-   readonly ILogger? logger;
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<IDataPacket> source, string? name = null, ILogger? logger = null) {
+   public ObservableViewStream(IObservable<DataPacket> source) =>
       this.source = source;
-      this.name = name;
-      this.logger = logger;
-   }
 
    public void Dispose() {
       GC.SuppressFinalize(this);
@@ -30,13 +23,7 @@ public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDispo
 
    protected override void myOnFirstSubscriber() {
       Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
-
-      void OnError(Exception exception) =>
-         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
-
-      void OnCompleted() =>
-         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
+      subscription = source.Subscribe(myFireEvent);
    }
 
    protected override void myOnLastSubscriber() {
@@ -45,6 +32,6 @@ public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDispo
       subscription = null;
    }
 
-   IDisposable IObservable<IDataPacket>.Subscribe(IObserver<IDataPacket> observer) =>
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
       source.Subscribe(observer);
 }

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,25 +1,14 @@
-using bts;
-
-using Bts.DataStream;
-using Bts.DataStreamCore;
-using Bts.DataStreamExpressions;
-using Bts.PubSub;
-
-using Microsoft.Extensions.Hosting;
-
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Reactive.Linq;
-using System.Threading;
-using System.Threading.Tasks;
+
+using Bts.DataStreamExpressions;
+using BCG.Utilities.General;
+using bts;
 
 namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IHostedService {
+   public class RemoteQueryManager : IWorker {
       public class StreamPubUnit {
          public string name;
          public string stream_expression;
@@ -50,45 +39,23 @@ namespace BCG {
       }
       private IDictionary<string, StreamPubUnit>
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      readonly IPubTableBroker _broker;
-      readonly IManagedSubscriber _sub;
-      readonly object syncRoot = new();
-      IDisposable? cloudStreamsSub;
-      IDisposable? remoteRequestsSub;
-      bool isDisposed;
+      private ViewStream _stream = new TableStream("cloud_streams");
+      private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
-         _sub = managedSubscriber;
-         _broker = pubTableBroker;
+      public RemoteQueryManager() {
+         _stream.onChange += _on_str;
+         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
       }
-
-      public Task StartAsync(CancellationToken cancellationToken) {
-         lock (syncRoot) {
-            Log.assert(!isDisposed);
-            Log.isnull(cloudStreamsSub);
-            Log.isnull(remoteRequestsSub);
-            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
-            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
-               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-            });
+      private void _on_str(object obj)
+      {
+         if (obj is not DataPacket dp) {
+            return;
          }
-         return Task.CompletedTask;
-      }
-
-      public Task StopAsync(CancellationToken cancellationToken) {
-         lock (syncRoot) {
-            if (!isDisposed) {
-               isDisposed = true;
-               Disposable.dispose(ref cloudStreamsSub);
-               Disposable.dispose(ref remoteRequestsSub);
-            }
-         }
-         return Task.CompletedTask;
-      }
-
-      void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
@@ -105,10 +72,7 @@ namespace BCG {
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
          if (persist) {
-            string tempDir = Path.GetTempPath();
-        Directory.CreateDirectory(tempDir); // make sure it exists
-
-        stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
+            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
          StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
@@ -118,5 +82,9 @@ namespace BCG {
       private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
+
+      public void doWork() {
+
+      }
    }
 }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -9,7 +9,7 @@ namespace BCG {
    [DataStreamUserRespresentation("remoteQuery")]
    public class RemoteQueryStream : ViewStream {
       private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = PublisherCore.Create();
+      private PublisherCore _publisher_core = new PublisherCore();
       private ViewStream _source_str;
 
       public RemoteQueryStream(params string[] lines) {

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -2,6 +2,9 @@ namespace bts {
    using System.IO;
    using BCG;
 
+   using System.IO;
+   using System.Threading.Tasks.Dataflow;
+
    public class RemoteRequest {
       public readonly string stream_expression;
       public readonly string published_table_name;
@@ -25,4 +28,4 @@ namespace bts {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,46 +1,33 @@
-using Spark.Core.Collections;
-
 using System;
-using System.Collections.Generic;
-using System.Linq;
 
-namespace BCG {   
+namespace BCG {
    public class AddColumnsStream : ViewStream {
-      public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
          string[] toks = exp.Split(':');
-         string col_name = toks[0].Trim();
-         string col_value = toks[1].Trim();
+         string col_name = toks[0];
+         string col_value = toks[1];
          Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return (col_name, tup.Item1, tup.Item2);
+         return Tuple.Create(col_name, tup.Item1, tup.Item2);
       }
 
-      public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
-         exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
-
-      readonly List<(string Name, Type Type, object Value)> _new_col_map;
-      IBlueTableSchema? _out_schema;
+      private Tuple<string, Type, object> _new_col;
+      private IBlueTableSchema _out_schema;
 
       public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col_map = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, _get_schema, _map);
+         _new_col = parse_exp(col_def);
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
          mySetRealSource(rs);
       }
 
       private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null) {
-            _out_schema = input_schema;
-            foreach (var m in _new_col_map) {
-               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
-            }
-         }
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
          return _out_schema;
       }
 
       private IBlueRow _map(IBlueRow r) {
          MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         foreach (var m in _new_col_map) {
-            rv.setValue(m.Name, m.Value);
-         }
+         rv.setValue(_new_col.Item1, _new_col.Item3);
          return rv;
       }
    }

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs

@@ -1,10 +1,9 @@
-using mbts;
-
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
 
 namespace BCG
 {
@@ -34,7 +33,7 @@ namespace BCG
 
          private void _stream_onChange(object newValue)
          {
-            var dp = (IDataPacket)newValue;
+            DataPacket dp = newValue as DataPacket;
             if (dp != null && dp.newRows.Count > 0)
             {
                _stream.onChange -= _stream_onChange;
@@ -43,8 +42,8 @@ namespace BCG
          }
       }
 
-      public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
-      public delegate string GetColumnPrefix(object?[] transposing_col_values);
+      public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
+      public delegate string GetColumnPrefix(object[] transposing_col_values);
 
       public delegate string GetColumnSuffix(string col_name);
       private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
@@ -52,31 +51,33 @@ namespace BCG
       private IncrementValues _increment_values;
       private GetColumnPrefix _get_column_prefix;
 
-      IBlueTableSchema? _schema;
+      private IBlueTableSchema _schema;
 
       private string[] _group_by_cols;
       private Pair<string, Type>[] _cols_per_transposed;
       private string[] _transposing_cols;
 
-      int[]? _transposing_col_indices;
+      private int[] _transposing_col_indices;
 
-      Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
+      private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
          : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
                 get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
       { }
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
       {
          if (get_column_suffix_del != null)
             _get_column_suffix = get_column_suffix_del;
          if (transposed_col_value_comparer != null)
          {
-            _transpsed_col_value_comparer_wrapper = new((p1, p2) =>
-               transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
+            _transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
+               {
+                  return transposed_col_value_comparer(p1.second, p2.second);
+               });
          }
          _group_by_cols = group_by_cols.Split(' ');
          _transposing_cols = transposed_cols_group.Split(' ');
@@ -105,7 +106,7 @@ namespace BCG
             col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
          foreach (string tc_prefix in tc_prefixes)
             foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
          IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
          if (_schema == null)
             _schema = schema;
@@ -129,14 +130,19 @@ namespace BCG
 
       public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
       {
-         var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
-         var group_by_cols_value_dict = new Dictionary<int, object?[]>();
+         Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
+         Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
          BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
          Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
          string table_name = string.Empty;
          if (grouped_rows.Count > 0)
          {
-            var schema = grouped_rows.Values.First()[0].schema;
+            IBlueTableSchema schema = null;
+            foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
+            {
+               schema = kvp.Value[0].schema;
+               break;
+            }
             table_name = schema.tableName;
             _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
             int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
@@ -149,22 +155,23 @@ namespace BCG
          else
             return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); 
          Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
-         var tc_values = new List<Pair<int, object?[]>>();
+         List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
          foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
          {
             //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values 
-            var incrementing_values = new Dictionary<int, object?[]>();
+            Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
             foreach (IBlueRow r in kvp.Value)
             {
                int hash = hasher.get_hash(r, _transposing_col_indices);
-               if (!incrementing_values.TryGetValue(hash, out var values))
+               object[] values;
+               if (!incrementing_values.TryGetValue(hash, out values))
                {
                   incrementing_values.Add(hash, _get_per_transposing_col_values(r));
                   string prefix = _get_column_prefix(_get_transposing_col_values(r));
                   if (!tc_prefixes.ContainsKey(hash))
                   {
                      tc_prefixes.Add(hash, prefix);
-                     tc_values.Add(new(hash, _get_transposing_col_values(r)));
+                     tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
                   }
                }
                else
@@ -187,18 +194,18 @@ namespace BCG
          _set_schema(prefixes, group_by_col_info, table_name);
          BlueTable result = new BlueTable(_schema);
          int id = 1;
-         foreach (var kvp in all_tc_infos_and_values)
+         foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
          {
             MutableBlueRow mbr = new MutableBlueRow(_schema);
             mbr.setValue("id", id++);
             for (int i = 0; i < _group_by_cols.Length; i++)
                mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
-            foreach (var tc_cols in kvp.Value)
+            foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
             {
                string prefix = tc_prefixes[tc_cols.Key];
                for (int i = 0; i < _cols_per_transposed.Length; i++)
                {
-                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
+                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
                   if(mbr.schema.columnNames.Contains(column))
                      mbr.setValue(column, tc_cols.Value[i]);
                }
@@ -210,25 +217,28 @@ namespace BCG
 
       #endregion
 
-      object?[] _get_transposing_col_values(IBlueRow row) {
-         var values = new object?[_transposing_cols.Length];
+      private object[] _get_transposing_col_values(IBlueRow row)
+      {
+         object[] values = new object[_transposing_cols.Length];
          for (int i = 0; i < _transposing_cols.Length; i++)
             values[i] = row[_transposing_cols[i]];
          return values;
       }
 
-      object?[] _get_per_transposing_col_values(IBlueRow row) {
-         var values = new object?[_cols_per_transposed.Length];
+      private object[] _get_per_transposing_col_values(IBlueRow row)
+      {
+         object[] values = new object[_cols_per_transposed.Length];
          for (int i = 0; i < _cols_per_transposed.Length; i++)
-            values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
+            values[i] = row[_cols_per_transposed[i].first];
          return values;
       }
 
-      object?[] _get_group_by_cols_values(IBlueRow row) {
-         var values = new object?[_group_by_cols.Length];
+      private object[] _get_group_by_cols_values(IBlueRow row)
+      {
+         object[] values = new object[_group_by_cols.Length];
          for (int i = 0; i < values.Length; i++)
             values[i] = row[_group_by_cols[i]];
          return values;
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -1,19 +1,22 @@
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
 namespace BCG
 {
    public class DerivativeStream : ViewStream 
    {
-      IBlueTableSchema? mySchema;
+
+      private IBlueTableSchema mySchema;
       private string myDerivandColumn;
       private string myOutputColumn;
       private string myKeyColumn;
       private DateTime myLastPacketArrival = DateTime.MinValue;      
       private ViewStream myDataSource;
-      Dictionary<string, double>? myPreviousValues;
+      
+      private IDictionary<string, double> myPreviousValues;
+
 
       public DerivativeStream(ViewStream input, string keyColumn,  string columnToDifferentiate, string outputColName)
       {
@@ -36,9 +39,9 @@ namespace BCG
 
       void myDataSource_onChange(object newValue)
       {
-         var dp = (IDataPacket)newValue;
+         DataPacket dp = (DataPacket)newValue;
          if (myLastPacketArrival == DateTime.MinValue)
-            mySchema = myBuildOutputSchema(dp.Schema);
+            mySchema = myBuildOutputSchema(dp.schema);
          else
          {
             DateTime now = DateTime.Now;
@@ -52,7 +55,7 @@ namespace BCG
             else
             {
                IBlueRowCollection deleted, added;
-               BlueDataUtility.computeDifferences(thisVersion, ((IDataPacket)myLastValue).newRows, out deleted, out added);
+               BlueDataUtility.computeDifferences(thisVersion, ((DataPacket)myLastValue).newRows, out deleted, out added);
                myFireEvent(new DataPacket(thisVersion, added, deleted));  
             }
          }
@@ -61,13 +64,12 @@ namespace BCG
 
       protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
       {
-         ExceptionAssert.NotNull(mySchema, myPreviousValues);
          object[] outValues = new object[mySchema.columnInfo.Length];
          r.Values.CopyTo(outValues, 0);
 
-         var derivand = r[myDerivandColumn];
-         var key = getRowKey(r);
-         if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
+         object derivand = r[myDerivandColumn];
+         double lastValue = default(double);
+         if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value)
             outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
          else
             outValues[mySchema.columnInfo.Length - 1] = 0.0;
@@ -82,19 +84,16 @@ namespace BCG
          cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double));
          return new BlueTableSchema("derviative of " + inSchema.tableName, cols);
       }
-
-      protected void myStoreNewPacketInfo(IDataPacket dp) {
+      protected void myStoreNewPacketInfo(DataPacket dp)
+      {
          myPreviousValues = new Dictionary<string, double>();
          myLastPacketArrival = DateTime.Now;
          foreach (IBlueRow row in dp.newRows)
          {
-            var derivand = row[myDerivandColumn];
+            object derivand = row[myDerivandColumn];
             if (derivand != null && derivand != DBNull.Value)
-               myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
+               myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand);
          }
       }
-
-      string getRowKey(IBlueRow row) =>
-         row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
    }
 }

src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs

@@ -1,9 +1,7 @@
-using mbts;
-
-using Spark.Core.Collections;
-
-using System;
+using System;
+using System.Collections;
 using System.Collections.Generic;
+using mbts;
 
 namespace BCG
 {
@@ -14,9 +12,9 @@ namespace BCG
    {
       ViewStream myLStream, myRStream;
 
-      DataPacket? myLastR;
-      DataPacket? myLastL;
-      BlueTable? myLastFired;
+      DataPacket myLastR = null, myLastL = null;
+
+      BlueTable myLastFired = null;
 
       double myThreshold = .00005;
 
@@ -26,14 +24,14 @@ namespace BCG
       public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : 
          this(null, v1, v2, thresh, ColumnsToMatch) { }
 
-      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
       {
          myThreshold = thresh;
          myLStream = v1;
          myRStream = v2;
          myColumnsToMatch = columnsToMatch;
       }
-      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
          : this(db,v1,v2,.00005,columnsToMatch)
       {
       }
@@ -52,22 +50,45 @@ namespace BCG
          myRStream.onChange -= myRChange;
       }
 
+      struct ListComparer : IEqualityComparer<List<object>>
+      {
+         public bool Equals(List<object> x, List<object> y)
+         {
+            if (x.Count != y.Count)
+               return false;
+
+            for (int i = 0; i < x.Count; ++i)
+               if (!x[i].Equals(y[i]))
+                  return false;
+            return true;
+         }
+
+         public int GetHashCode(List<object> obj)
+         {
+            int ret = 0;
+            for(int i = 0; i < obj.Count; ++i)
+               ret ^= obj[i].GetHashCode();
+            return ret;
+         }
+      }
+
       void myHandleCallbacks()
       {
-         DataPacket? packet;
+         DataPacket packet = null;
+
          if (myLastL == null || myLastR == null)
          {
             packet = myLastL == null ? myLastR : myLastL;
-            if (packet is not null) {
+            if (packet != null)
                myFireEvent(packet);
-               myLastFired = new BlueTable(packet.newRows);
-            }
+
+            myLastFired = new BlueTable(packet.newRows);
             return;
          }
 
          BlueTable table = new BlueTable(myLastL.newRows.schema);
 
-         var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
+         Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
 
          foreach(BlueRow row in myLastL.newRows)
          {
@@ -97,7 +118,11 @@ namespace BCG
                }
                catch (Exception) { }
             }
-            rowPairs.TryGetValue(hash, out var pair);
+
+            Pair<BlueRow, BlueRow> pair;
+            rowPairs.TryGetValue(hash, out pair);
+
+
             if (pair == null)
                pair = new Pair<BlueRow, BlueRow>(null, row);
             else
@@ -113,7 +138,8 @@ namespace BCG
 
             if (pair.first == null || pair.second == null)
             {
-               var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
+               BlueRow row = pair.first == null ? pair.second : pair.first;
+
                Dictionary<string, object> newRow = new Dictionary<string, object>();
                foreach (ColumnInfo info in row.schema.columnInfo)
                {
@@ -185,4 +211,4 @@ namespace BCG
 
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs

@@ -5,7 +5,7 @@ namespace BCG
 {
 	public class FileViewStream : ViewStream
 	{
-      DataPacket? myDp;
+      private DataPacket myDp;
 		public FileViewStream (string dataFileName, 
          string schemaFileName)
          : this (dataFileName, schemaFileName, Int32.MaxValue)

src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs

@@ -1,7 +1,8 @@
-using BCG.Utilities.General;
-
-using System;
-
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 namespace BCG
 {
     public class DateBasedFilter: IDynamicStreamFilter, IWorker
@@ -25,14 +26,20 @@ namespace BCG
             myReference = myTodaysDate.AddDays(-1 * myNDays);                     
         }       
         
-        public event FilterChangeDelegate? onFilterChange;       
+        public event FilterChangeDelegate onFilterChange;       
         
-        public bool filter(IBlueRow r) =>
-           r[myDateColumn] switch {
-              DBNull or null => true,
-              DateTime t => t >= myReference,
-              _ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
-           };
+        public bool filter(IBlueRow r)
+        {
+            if (DBNull.Value.Equals(r[myDateColumn]))
+            {
+                //return false;
+                return true; //it can be considered as our assumption  whenever datecolumn is db null we return true 
+            }
+            DateTime t = (DateTime)r[myDateColumn];                   
+            bool returnVallue =  t < myReference ? false : true;
+            return returnVallue;
+        }
+              
 
         public void doWork()
         {
@@ -40,7 +47,7 @@ namespace BCG
             {
                 myTodaysDate = DateTime.Today;
                 myReference = myTodaysDate.AddDays(-1 * myNDays);
-                onFilterChange?.Invoke();
+                onFilterChange();
             }         
         }      
     }

src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs

@@ -1,6 +1,8 @@
-using BCG.Utilities.General;
-
-using System;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 
 namespace BCG
 {
@@ -24,7 +26,7 @@ namespace BCG
 
       #region IDynamicStreamFilter Members
 
-      public event FilterChangeDelegate? onFilterChange;
+      public event FilterChangeDelegate onFilterChange;
 
       #endregion
 
@@ -32,10 +34,7 @@ namespace BCG
 
       public bool filter(IBlueRow r)
       {
-         DateTime rowDate = r[myDateColName] switch {
-            DBNull or null => DateTime.Now.Date,
-            object obj => (DateTime)obj,
-         };
+         DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date;
          DateTime today = DateTime.Now.Date;
          return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
       }
@@ -49,7 +48,7 @@ namespace BCG
          if (DateTime.Now.Date != myLastCheckedTime.Date)
          {
             myLastCheckedTime = DateTime.Now;
-            onFilterChange?.Invoke();
+            onFilterChange();
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,23 +1,15 @@
-using bts;
+using Bts.DataStream;
 
-using Bts.DataStream.Primitives.Filter;
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-using Spark.Core.Disposables;
+using mbts.config;
 
 using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
 using System.Text.RegularExpressions;
 
 namespace BCG.StreamWorld.Addition {
    [DataStreamUserRespresentation(false)]
    public class FilterStream : ViewStream {
       private ViewStream myBaseStream;
-      BlueTable? myBaseTable;
-      BlueTable? myStateTable;
+      private BlueTable myBaseTable, myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
@@ -29,9 +21,8 @@ namespace BCG.StreamWorld.Addition {
       public FilterStream(ViewStream vs, IStreamFilter filter) {
          myBaseStream = vs;
          myFilter = filter;
-         vs.ErrorChanged += OnError;
-         if (filter is IDynamicStreamFilter dfilter)
-            dfilter.onFilterChange += myHandleFilterChange;
+         if (filter is IDynamicStreamFilter)
+            (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
       }
 
 
@@ -41,69 +32,46 @@ namespace BCG.StreamWorld.Addition {
 
       protected override void myOnLastSubscriber() {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
-         myBaseTable?.clear();
-         myStateTable?.clear();
-      }
-
-      bool filterRow(IBlueRow r) {
-         try {
-            return myFilter.filter(r);
-         } catch (Exception ex) {
-            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
-         }
-         return false;
+         myBaseTable.clear();
+         myStateTable.clear();
       }
 
       private void myHandleChanges(object newValue) {
-         try {
-            var dp = (IDataPacket)newValue;
-            var schema = dp.Schema;
-            init(schema);
-            var dels = new BlueTable(schema);
-            var adds = new BlueTable(schema);
-            foreach (IBlueRow r in dp.deletedRows) {
-               myBaseTable.deleteRow(r);
-               if (myStateTable.containsID(r.Id())) {
-                  dels.addRow(r);
-                  myStateTable.deleteRow(r);
-               }
-            }
-            foreach (IBlueRow r in dp.addedRows) {
-               myBaseTable.addRow(r);
-               if (filterRow(r)) {
-                  adds.addRow(r);
-                  myStateTable.addRow(r);
-               }
-            }
-            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
-               myFireEvent(new DataPacket(myStateTable, adds, dels));
-         } catch (Exception ex) {
-            OnError(ex);
+         DataPacket dp = newValue as DataPacket;
+         if (myStateTable == null) {
+            bts.Log.assert(myBaseTable == null);
+            myStateTable = new BlueTable(dp.schema);
+            myBaseTable = new BlueTable(dp.schema);
          }
+         BlueTable dels = new BlueTable(dp.schema),
+                     adds = new BlueTable(dp.schema);
+         foreach (IBlueRow r in dp.deletedRows) {
+            myBaseTable.deleteRow(r);
+            if (myStateTable.containsID((int)r["id"])) {
+               dels.addRow(r);
+               myStateTable.deleteRow(r);
+            }
+         }
+         foreach (IBlueRow r in dp.addedRows) {
+            myBaseTable.addRow(r);
+            if (myFilter.filter(r)) {
+               adds.addRow(r);
+               myStateTable.addRow(r);
+            }
+         }
+         if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
+            myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
       public IStreamFilter filter {
          set {
             myFilter = value;
             if (myBaseTable != null) {
-               ExceptionAssert.NotNull(myStateTable);
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
-               // If the filter throws an exception, we can rethrow it as a StreamFilterException
-               // and include the 'currentRow' (for which the filter failed) to provide context.
-               IBlueRow? currentRow = null;
-               foreach (IBlueRow r in myBaseTable) {
-                  try {
-                     currentRow = r;
-                     if (filterRow(r)) {
-                        myStateTable.addRow(r);
-                     }
-                  } catch (Exception ex) {
-                     Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-                     OnError(new StreamFilterException(StreamId, StreamName, null, ex, currentRow));
-                  }
-               }
+               foreach (IBlueRow r in myBaseTable)
+                  if (myFilter.filter(r))
+                     myStateTable.addRow(r);
                IBlueRowCollection dels, adds;
                BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
                myFireEvent(new DataPacket(myStateTable, adds, dels));
@@ -115,17 +83,6 @@ namespace BCG.StreamWorld.Addition {
       //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
 
       private void myHandleFilterChange() { filter = myFilter; }
-
-      [MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
-      void init(IBlueTableSchema schema) {
-         if (myStateTable is null) {
-            Log.isnull(myBaseTable);
-            myStateTable = new BlueTable(schema);
-            myBaseTable = new BlueTable(schema);
-         } else {
-            Log.notnull(myBaseTable);
-         }
-      }
    }
 }
 
@@ -149,24 +106,29 @@ namespace BCG
       }
 
       #region IStreamFilter Members
-      public bool filter(IBlueRow dr) =>
-         myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
-            DBNull or null => string.Empty,
-            object obj => obj.ToString() ?? string.Empty
-         }));
+      public bool filter(IBlueRow dr)
+      {
+         bool rv = true;
+         for (int i = 0; i < myCols.Length; i++)
+         {
+            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+            if (!rv)
+               return rv;
+         }
+         return rv;
+      }
       #endregion
    }
 
    public class FilterStream : ViewStream
    {
-      BlueTable? myLastFiredRows;
-      IBlueRowCollection? myLastReceivedRows;
-      IStreamFilter? myFilter;
-      readonly ViewStream? myParentStream;
-      string? myFilterString;
-      readonly StreamWorld.Addition.FilterStream? myFilterStream;
-      readonly bool myUseNewFilter = false;
-      IDisposable? subscription;
+      private BlueTable myLastFiredRows;
+      private IBlueRowCollection myLastReceivedRows;
+      private IStreamFilter myFilter;
+      private ViewStream myParentStream;
+      private string myFilterString;
+      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
+      private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -196,10 +158,9 @@ namespace BCG
          {
             myParentStream = vs;
             myFilter = filter;
-            myFilterString = filter.ToString() ?? string.Empty;
-            if (filter is IDynamicStreamFilter dfilter) {
-               dfilter.onFilterChange += myHandleFilterChange;
-            }
+            myFilterString = filter.ToString();
+            if (filter is IDynamicStreamFilter)
+               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
          }
       }
 
@@ -223,20 +184,18 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber() {
-         ExceptionAssert.IsNull(subscription);
-         subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
-      }
+      protected override void myOnFirstSubscriber()
+      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         Disposable.Dispose(ref subscription);
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
       }
 
       protected override void myFireEvent(object o)
       {
-         var dp = (IDataPacket)o;
+         DataPacket dp = o as DataPacket;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -245,39 +204,34 @@ namespace BCG
       private void myHandleCallbacks(object newValue)
       {
          DataPacket retVal;
-         var arrivedPacket = (IDataPacket)newValue;
+         DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
-         try {
-            if (myLastFiredRows != null)
-            {
-               var addedRows = new BlueTable(myLastFiredRows.schema);
-               var deletedRows = new BlueTable(myLastFiredRows.schema);
-               retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
-               foreach (IBlueRow r in arrivedPacket.deletedRows)
-                  if (myLastFiredRows.containsID(r.Id()))
-                  {
-                     myLastFiredRows.deleteRow(r);
-                     deletedRows.addRow(r);
-                  }
-               foreach (IBlueRow r in arrivedPacket.addedRows)
-                  if (filterRow(r))
-                  {
-                     myLastFiredRows.addRow(r);
-                     addedRows.addRow(r);
-                  }
-            }
-            else
-            {
-               myLastFiredRows = new BlueTable(arrivedPacket.Schema);
-               retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-               foreach (IBlueRow r in arrivedPacket.newRows)
-                  if (filterRow(r))
-                     myLastFiredRows.addRow(r);
-            }
-            myFireEvent(retVal);
-         } catch (Exception ex) {
-            OnError(ex);
+         if (myLastFiredRows != null)
+         {
+            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+               new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.deletedRows)
+               if (myLastFiredRows.containsID((int)r["id"]))
+               {
+                  myLastFiredRows.deleteRow(r);
+                  (retVal.deletedRows as BlueTable).addRow(r);
+               }
+            foreach (IBlueRow r in arrivedPacket.addedRows)
+               if (myFilter.filter(r))
+               {
+                  myLastFiredRows.addRow(r);
+                  (retVal.addedRows as BlueTable).addRow(r);
+               }
          }
+         else
+         {
+            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.newRows)
+               if (myFilter.filter(r))
+                  myLastFiredRows.addRow(r);
+         }
+         myFireEvent(retVal);
       }
 
       public IStreamFilter filter
@@ -286,7 +240,7 @@ namespace BCG
          {
             if (myUseNewFilter)
             {
-               requireNewFilter().filter = filter;
+               myFilterStream.filter = filter;
             }
             else
             {
@@ -295,29 +249,25 @@ namespace BCG
                {
                    if (myLastReceivedRows != null)
                    {
-                     try {
                        BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
                        foreach (IBlueRow r in myLastReceivedRows)
-                           if (filterRow(r))
+                           if (myFilter.filter(r))
                                nRows.addRow(r);
                        IBlueRowCollection dRows = new BlueTable(nRows.schema),
                           aRows = new BlueTable(nRows.schema);                      
                        BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
                        myLastFiredRows = nRows;
                        myFireEvent(new DataPacket(nRows, aRows, dRows));
-                     } catch (Exception ex) {
-                        OnError(ex);
-                     }
-                  }
+                   }
                }
             }
          }
          get
          {
             if (myUseNewFilter)
-               return requireNewFilter().filter;
+               return myFilterStream.filter;
             else
-               return requireOldFilter();
+               return myFilter;
          }
       }
 
@@ -330,25 +280,7 @@ namespace BCG
 
       private void myHandleFilterChange()
       {
-         filter = requireOldFilter();
+         filter = myFilter;
       }
-
-      bool filterRow(IBlueRow r) {
-         try {
-            return requireOldFilter().filter(r);
-         } catch (Exception ex) {
-            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
-         }
-         return false;
-      }
-
-      [MemberNotNull(nameof(myFilterStream))]
-      StreamWorld.Addition.FilterStream requireNewFilter() =>
-         ExceptionAssert.MemberNotNull(myFilterStream);
-
-      [MemberNotNull(nameof(myFilter))]
-      IStreamFilter requireOldFilter() =>
-         ExceptionAssert.MemberNotNull(myFilter);
    }
 }

src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs

@@ -1,6 +1,8 @@
-using BCG.Utilities.General;
-
-using System;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 
 namespace BCG
 {    
@@ -27,7 +29,7 @@ namespace BCG
             myReference = myCurrentTime.AddSeconds(-1 * myNSecs);                  
         }
         
-        public event FilterChangeDelegate? onFilterChange;       
+        public event FilterChangeDelegate onFilterChange;       
         
         public bool filter(IBlueRow r)
         {
@@ -36,8 +38,9 @@ namespace BCG
                 //it can be considered as our assumption  whenever datecolumn is db null we return true 
                 return true;  //dont use this for volparamsched use true              
             }
-            var t = (DateTime)r[myDateColumn]!;
-            return t >= myReference;
+            DateTime t = (DateTime)r[myDateColumn];                   
+            bool returnVallue =  t < myReference ? false : true;
+            return returnVallue;
         }
               
 
@@ -48,7 +51,8 @@ namespace BCG
                 myCurrentTime = DateTime.Now;
                 myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
                 myLastTimeFilteredApplied = DateTime.Now;              
-                onFilterChange?.Invoke();
+                onFilterChange();
+                
             }           
         }
     }

src/DataStreamSuite/DataStream/Primitives/Filter/StreamFilterException.cs

@@ -1,19 +0,0 @@
-#nullable enable
-namespace Bts.DataStream.Primitives.Filter;
-
-using BCG;
-
-using System;
-
-public class StreamFilterException : Exception {
-   public StreamFilterException(Guid streamId, string streamName, string? message, Exception? innerException, IBlueRow? badRow = null)
-      : base(message, innerException) {
-      StreamId = streamId;
-      StreamName = streamName;
-      BadRow = badRow;
-   }
-
-   public Guid StreamId { get; }
-   public string StreamName { get; }
-   public IBlueRow? BadRow { get; }
-}

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -52,14 +52,14 @@ namespace BCG {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
          myVsd = vsd;
          _time_col = rowTimeColumnName;
          _ttl = rowTimeToLive;
-         _filter_period = filter_period;
+         myFilterFrequency = filterFrequency;
          myNextFilter = DateTime.MinValue;
 
          // add worker
@@ -71,10 +71,11 @@ namespace BCG {
 
       #region IStreamFilter
       public bool filter(IBlueRow row) {
-         DateTime now_utc = DateTime.UtcNow;
-         DateTime row_time_utc = row.get_datetime(_time_col, DateTime.Now).ToUniversalTime();
-         if (now_utc > row_time_utc) {
-            TimeSpan span = now_utc - row_time_utc;
+         DateTime now = DateTime.Now;
+
+         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
+         if (now > row_time) {
+            TimeSpan span = now - row_time;
             if (span >= _ttl) {
                return (false);
             }
@@ -89,9 +90,9 @@ namespace BCG {
 
          if (now > myNextFilter) {
             // fire event
-            onFilterChange?.Invoke();
+            onFilterChange();
 
-            myNextFilter = now.Add(_filter_period);
+            myNextFilter = now.Add(myFilterFrequency);
          }
       }
       #endregion
@@ -107,7 +108,7 @@ namespace BCG {
       private ViewStreamDatabase myVsd;
       private string _time_col;
       private TimeSpan _ttl;
-      private TimeSpan _filter_period;
+      private TimeSpan myFilterFrequency;
       private DateTime myNextFilter;
       #endregion
    }

src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs

@@ -1,13 +1,11 @@
-using BCG.Utilities.General;
-
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 
 using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
+using System.Diagnostics;
 
 namespace BCG
 {
@@ -42,12 +40,12 @@ namespace BCG
       private IComparer<IBlueRow> myRightCompare;
       protected FuzzyRowComparer myFuzzyCompare;
       protected TimeSpan myTimeWindow;
-      protected IBlueTableSchema? myLeftSchema;
-      protected IBlueTableSchema? myRightSchema;
-      protected IBlueTableSchema? myOutputSchema;
+      protected IBlueTableSchema myLeftSchema;
+      protected IBlueTableSchema myRightSchema;
+      protected IBlueTableSchema myOutputSchema;
       private int myID = 0;
 
-      protected TimeWindowTable? myStateRows;
+      protected TimeWindowTable myStateRows;
 
       private DateTime myNextCull;
       const long MinCullIntervalMS = 1000;
@@ -67,7 +65,7 @@ namespace BCG
          this.myFuzzyCompare = fuzzyCompare;
       }
 
-      public virtual IBlueTableSchema? OutputSchema
+      public virtual IBlueTableSchema OutputSchema
       {
          get
          {
@@ -152,7 +150,7 @@ namespace BCG
          int count;
          for (count = 0; count < list.Count; count++)
          {
-            if ((DateTime)list[count][timeColumn]! > expiry)
+            if ((DateTime)list[count][timeColumn] > expiry)
             {
                // Stop when we find the first row that we can keep.  Since this is sorted all others are also kept.
                break;
@@ -174,16 +172,16 @@ namespace BCG
             this.myFitness = fitness;
          }
          public IBlueRow myRightRow { get; private set; }
-         public IBlueRow? myLeftRow { get; set; }
+         public IBlueRow myLeftRow { get; set; }
          public double myFitness { get; private set; }
-         public IBlueRow? myMergedRow { get; set; }
+         public IBlueRow myMergedRow { get; set; }
 
          #region IComparable<RowMatch> Members
 
-         public int CompareTo(RowMatch? other) =>
-            other is null
-               ? 1
-               : myFitness.CompareTo(other.myFitness);
+         public int CompareTo(RowMatch other)
+         {
+            return myFitness.CompareTo(other.myFitness);
+         }
 
          #endregion
       }
@@ -264,12 +262,12 @@ namespace BCG
 
       private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
       {
-         ExceptionAssert.NotNull(myStateRows);
          // We already emitted correction.MergedRow so we need to redact that and 
          // then emit correction.left as an add since it is no longer matched
          foreach (RowMatch correction in corrections.Distinct())
          {
-            myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow));
+            Debug.Assert(correction.myMergedRow != null);
+            this.myStateRows.RemoveRow(correction.myMergedRow);
             adds.addRow(this.MergeRows(correction.myLeftRow, null));
             dels.addRow(correction.myMergedRow);
          }
@@ -287,7 +285,7 @@ namespace BCG
          int r = 0;
          for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
          {
-            if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!)
+            if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
             {
                // Advance Left
                IEnumerable<RowMatch> rm = rightMatches[l];
@@ -334,7 +332,7 @@ namespace BCG
 
       private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
       {
-         IBlueRow? merged = null;
+         IBlueRow merged = null;
          if (rightMatches != null)
          {
             foreach (RowMatch match in rightMatches)
@@ -368,9 +366,9 @@ namespace BCG
       /// Override this function to provide a custom row merge.
       /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
       /// </summary>
-      protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
+      protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
       {
-         Dictionary<string, object?> newRow = new();
+         Dictionary<string, object> newRow = new Dictionary<string, object>();
 
          if (left == null && right == null) throw new ArgumentException("oh come on.");
 
@@ -395,15 +393,14 @@ namespace BCG
             }
          }
          newRow["id"] = this.myID++;
-         newRow[NEW_TIME_COLUMN] = left is not null
-            ? left[myLeftTimeColumn]
-            : right![myRightTimeColumn];
+         newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
+
          return new BlueRow(this.OutputSchema, newRow);
       }
 
       private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
       {
-         List<RowMatch>? matches = null;
+         List<RowMatch> matches = null;
 
          // Search through all rows in list and compare with the fuzzy comparer
          for (int x = 0; x < list.Count; x++)
@@ -424,7 +421,8 @@ namespace BCG
                matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
 
                // Mark the best match for this particular right hand match
-               if (this.myBestMatch.TryGetValue(match, out var oldMatch))
+               RowMatch oldMatch;
+               if (this.myBestMatch.TryGetValue(match, out oldMatch))
                {
                   if (currentMatch.myFitness < oldMatch.myFitness)
                   {
@@ -443,7 +441,7 @@ namespace BCG
             }
          }
 
-         return matches ?? Enumerable.Empty<RowMatch>();
+         return matches;
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -69,7 +69,14 @@ namespace BCG
       protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
       #endregion
 
-      IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
-         new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project"));
+      private IBlueRow myProjectRow (IBlueRow r, string colMapExpression)
+      {
+         BlueTable auxTable = new BlueTable (r.schema);
+         auxTable.addRow (r);
+         BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable);
+         ProjectStream projStream = new ProjectStream (staticStream, colMapExpression);
+         return BlueDataUtility.extractRow (projStream.rows, 0);
+      }
+
 	}
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs

@@ -1,10 +1,14 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
 
 namespace BCG
 {
    public interface INonIncGroupByEvaluator
    {
-      IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-      string[]? get_group_by_cols();
+      IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+      string[] get_group_by_cols();
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,51 +1,46 @@
-using bts;
-using bts.SilverPine;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-using Spark.Core.Disposables;
-
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Text.Json;
+
+using bts;
+using bts.SilverPine;
 
 namespace BCG {
-   //[DataStreamUserRespresentation("AggregateStream")]
+   [DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]?> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>?> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
                                           Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable? rv = null;
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
-               var out_rows = _evaluate_func(g_rows);
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
                if (out_rows != null && out_rows.Count > 0) {
                   rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-                  rv.addRows(out_rows);
                }
+               rv.addRows(out_rows);
             }
             return rv;
          }
-         public string[]? get_group_by_cols() {
+         public string[] get_group_by_cols() {
             return _groupby_cols_func();
          }
 
       }
-      readonly ViewStream? _main_stream;
-      readonly INonIncGroupByEvaluator? _groupby_func;
-      BlueRowHasher? _row_hasher;
-      readonly string[]? _groupby_cols;
+      private ViewStream _main_stream;
+      private INonIncGroupByEvaluator _groupby_func;
+      private BlueRowHasher _row_hasher;
+      private string[] _groupby_cols;
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
-      IDisposable? subscription;
 
       public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -54,7 +49,7 @@ namespace BCG {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public NonIncGroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
@@ -63,7 +58,7 @@ namespace BCG {
 
       public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
-         Func<string[]?> groupby_cols_func = () => {
+         Func<string[]> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
@@ -80,17 +75,12 @@ namespace BCG {
       }
 
 
-      protected override void myOnFirstSubscriber() {
-         ExceptionAssert.IsNull(subscription);
-         ExceptionAssert.NotNull(_main_stream).Subscribe(_main_stream_onchange);
-      }
+      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
 
-      protected override void myOnLastSubscriber() =>
-         Disposable.Dispose(ref subscription);
+      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
 
       private void _main_stream_onchange(object nv) {
-         ExceptionAssert.NotNull(_groupby_func);
-         var dp = (IDataPacket)nv;
+         DataPacket dp = nv as DataPacket;
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -104,12 +94,8 @@ namespace BCG {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         var total_state = _groupby_func.evaluate(grouped_rows);
+         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
          _timer.Stop();
-         if (total_state is null) {
-            Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
-            return;
-         }
          var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;
          if (applyElapsedMs >= 50) {
@@ -118,12 +104,8 @@ namespace BCG {
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
          // TODO,
-         if (total_state != null) {
-            var deletedRows = myLastValue is null
-               ? new BlueTable(total_state.schema)
-               : ((IDataPacket)myLastValue).newRows;
-            myFireEvent(new DataPacket(total_state, total_state, deletedRows));
-         }
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs

@@ -1,6 +1,6 @@
 using System;
+using BCG.Utilities;
 using System.Collections;
-using System.Collections.Generic;
 
 namespace BCG
 {
@@ -11,10 +11,12 @@ namespace BCG
    /// </summary>
    public class RowDictionaryTable : IBlueRowCollection
    {
-      protected readonly Dictionary<string, BlueTable> myRowCollections = new();
-      protected readonly BlueTable myLinearCollection;
-      protected readonly string[] myCols;
+      protected IDictionary myRowCollections = new Hashtable ();
+      protected BlueTable myLinearCollection;
+      protected string[] myCols;
       protected IBlueTableSchema mySchema;
+      
+      protected RowDictionaryTable ()  {}
 
       public RowDictionaryTable (IBlueRowCollection rows, params string[] cols)
          : this (rows.schema, cols)
@@ -38,7 +40,7 @@ namespace BCG
 
       public virtual void addRow (IBlueRow row)
       {
-         var rows = getMutableGroup(row);
+         BlueTable rows = (BlueTable)getGroup (row);
          if (rows == null)
          {
             //i.e we dont have any group for this row => we need to create a new group
@@ -51,7 +53,7 @@ namespace BCG
 
       public virtual void removeRow (IBlueRow row)
       {
-         var rowGroup = requireMutableGroup(row);
+         BlueTable rowGroup = (BlueTable)getGroup (row);
 //         BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
          rowGroup.deleteRow (row);
          myLinearCollection.deleteRow (row);
@@ -63,7 +65,7 @@ namespace BCG
 
       public void removeGroup (IBlueRow representativeRow)
       {
-         var rowGroup = requireGroup(representativeRow);
+         IBlueRowCollection rowGroup = getGroup (representativeRow);
          // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
          foreach (IBlueRow row in rowGroup)
             myLinearCollection.deleteRow (row);
@@ -83,31 +85,24 @@ namespace BCG
       /// <returns></returns>
       //public IBlueRowCollection getGroup (IBlueRow row)  {return getGroup (
 
-      public IBlueRowCollection? getGroup (IBlueRow r) =>
-         getGroup(myRowToString(r));
+      public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));}
 
       public ICollection getGroups ()  {return myRowCollections.Values;}
 
-      public IBlueRowCollection? getGroup(string repString) =>
-         getMutableGroup(repString);
+      private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2)
+      {
+         bool retVal = true;
+         foreach (string col in myCols)
+            retVal &= r1[col].Equals(r2[col]);
+         return retVal;
+      }
 
-      BlueTable? getMutableGroup(IBlueRow row) =>
-         getMutableGroup(myRowToString(row));
-
-      BlueTable? getMutableGroup(string group) =>
-         myRowCollections.GetValueOrDefault(group);
-
-      IBlueRowCollection requireGroup(IBlueRow row) =>
-         requireGroup(myRowToString(row));
-
-      IBlueRowCollection requireGroup(string group) =>
-         getGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
-
-      BlueTable requireMutableGroup(IBlueRow row) =>
-         requireMutableGroup(myRowToString(row));
-
-      BlueTable requireMutableGroup(string group) =>
-         getMutableGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
+      public IBlueRowCollection getGroup (string repString)
+      {
+         IBlueRowCollection retVal = myRowCollections[repString] != null ?
+            (IBlueRowCollection) myRowCollections[repString] : null;
+         return retVal;
+      }
 
       private string myRowToString (IBlueRow r)  {return BlueDataUtility.rowToString (r, myCols);}
 

src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs

@@ -1,31 +1,31 @@
-//using System;
-//using System.Collections.Generic;
-//using System.Linq;
-//using System.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
-//namespace BCG
-//{
-//   public class FullJoinStream : ViewStream
-//   {
-//      private ViewStream myMainStream;
-//      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
-//         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
-//      {
-//         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
-//         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
-//         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
-//         myMainStream = new AppendStream(myMainStream, filtered);
-//      }
+namespace BCG
+{
+   public class FullJoinStream : ViewStream
+   {
+      private ViewStream myMainStream;
+      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
+         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
+      {
+         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
+         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
+         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
+         myMainStream = new AppendStream(myMainStream, filtered);
+      }
 
-//      protected override void myOnFirstSubscriber()
-//      {
-//         myMainStream.onChange += myHandleMainStreamChange;
-//      }
+      protected override void myOnFirstSubscriber()
+      {
+         myMainStream.onChange += myHandleMainStreamChange;
+      }
 
-//      private void myHandleMainStreamChange(object newData)
-//      {
-//         IBlueRowCollection result = (newData as DataPacket).newRows;
-//         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
-//      }
-//   }
-//}
+      private void myHandleMainStreamChange(object newData)
+      {
+         IBlueRowCollection result = (newData as DataPacket).newRows;
+         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BCG
 {
    [DataStreamUserRespresentation (false)]
@@ -15,7 +17,6 @@ namespace BCG
 
       protected override void myInitFiredPacket ()
       {
-         requireSchema();
          myLastFiredPacket = new DataPacket ();
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -24,10 +25,6 @@ namespace BCG
 
       public override void onTableClear (bool isLeft)
       {
-         if (myLastFiredPacket is null) {
-            return;
-         }
-         requireSchema();
          //incase of natural join, left or right wont matter, the end result zeroes out
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs

@@ -1,7 +1,5 @@
-using Spark.Core.Diagnostics;
-
 using System;
-using System.Diagnostics.CodeAnalysis;
+using System.Collections;
 
 namespace BCG
 {
@@ -15,10 +13,12 @@ namespace BCG
       protected JoinOperand myLeftOperand, myRightOperand;
       protected int myLeftJoinColOrdinal, myRightJoinColOrdinal;
       protected ViewStream myLeftStream, myRightStream;
-      protected JoinTableSchema? mySchema;
-      protected DataPacket? myLastFiredPacket;
+      protected JoinTableSchema mySchema;
+      protected DataPacket myLastFiredPacket;
       protected bool myOnceFired = false;
 
+      public IBlueTableSchema schema {get{return mySchema;}}
+
       public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight)
       {
          myLeftStream = vsLeft;
@@ -27,9 +27,6 @@ namespace BCG
          myRightOperand = new JoinOperand (jcRight, false);
       }
 
-      public IBlueTableSchema schema =>
-         requireSchema();
-
       protected override void myOnFirstSubscriber() 
       {
          myLeftOperand.master = this;
@@ -65,32 +62,29 @@ namespace BCG
          {
             myLeftJoinColOrdinal = myLeftOperand.joinColOridinal;
             myRightJoinColOrdinal = myRightOperand.joinColOridinal;
-            myLeftOperand.RequireInitialized();
-            myRightOperand.RequireInitialized();
             mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol, 
                myRightOperand.operandSchema, myRightOperand.joinCol);
             myInitFiredPacket ();
          }
       }
 
-      [MemberNotNull(nameof(mySchema), nameof(myLastFiredPacket))]
       protected virtual void myInitFiredPacket ()
       {
-         var schema = requireSchema();
          myLastFiredPacket = new DataPacket ();
-         myLastFiredPacket.newRows = new JoinTable (schema);
-         myLastFiredPacket.addedRows = new BlueTable (schema);
-         myLastFiredPacket.deletedRows = new BlueTable (schema);
+         myLastFiredPacket.newRows = new JoinTable (mySchema);
+         myLastFiredPacket.addedRows = new BlueTable (mySchema);
+         myLastFiredPacket.deletedRows = new BlueTable (mySchema);
       }
 
-      public IBlueTableSchema? getSchema (bool isLeft) =>
-         isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
+      public IBlueTableSchema getSchema (bool isLeft)
+      {
+         IBlueTableSchema retVal;
+         retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
+         return retVal;
+      }
 
       public void endSession ()
       {
-         if (myLastFiredPacket is null) {
-            return;
-         }
          if (!myOnceFired)
          {
             myOnceFired = true;
@@ -103,17 +97,13 @@ namespace BCG
 
       protected virtual void myInit ()
       {
-         myLeftOperand.Reset ();
-         myRightOperand.Reset ();
+         myLeftOperand.init ();
+         myRightOperand.init ();
          myLastFiredPacket = null;
       }
 
       public virtual void onTableClear (bool isLeft)
       {
-         if (myLastFiredPacket is null) {
-            return;
-         }
-         requireSchema();
          //incase of natural join, left or right wont matter, the end result zeroes out
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinTable (mySchema);
@@ -122,19 +112,15 @@ namespace BCG
 
       public virtual void deleteRows (IBlueRowCollection rows, bool isLeft)
       {
-         var last = myLastFiredPacket;
-         if (last is null) {
-            return;
-         }
          JoinOperand thisOp, otherOp;
          int joinColOrdinal;
          myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft);
          foreach (IBlueRow row in rows)
          {
             thisOp.deleteRow (row);
-            if (otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)).Count == 0)
+            if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0)
                continue;   //row does not participate in join
-            ((JoinTable)last.newRows).deleteRow(isLeft, row, last);
+            ((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket);
          }
       }
 
@@ -149,12 +135,10 @@ namespace BCG
             otherOp = myLeftOperand;
             joinColOrdinal = myRightJoinColOrdinal;
          }
-         var schema = requireSchema();
-         myLastFiredPacket ??= new DataPacket(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema));
          foreach (IBlueRow row in rows)
          {
             thisOp.addRow (row);
-            RowList matchedNodes = otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal));
+            RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]);
             if (matchedNodes.Count == 0)
                continue;   //this row did not participate in the join
             if (isLeft)
@@ -162,22 +146,10 @@ namespace BCG
                   ((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket);
             else
                foreach (RowNode matchedNode in matchedNodes)
-                  ((JoinTable)myLastFiredPacket.newRows).addRow(ExceptionAssert.NotNull(matchedNode.row), row, myLastFiredPacket);
+                  ((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket);
          }
       }
 
-      [MemberNotNull(nameof(mySchema))]
-      protected JoinTableSchema requireSchema() =>
-         ExceptionAssert.NotNull(mySchema);
-
-      [MemberNotNull(nameof(myLastFiredPacket))]
-      protected DataPacket requireLastFiredPacket() {
-         if (myLastFiredPacket is null) {
-            myInitFiredPacket();
-         }
-         return myLastFiredPacket;
-      }
-
       private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft)
       {
          if (isLeft)
@@ -194,9 +166,6 @@ namespace BCG
          }
       }
 
-      object getJoinKey(IBlueRow row, int joinColOrdinal) =>
-         row[joinColOrdinal] ?? throw new ArgumentException("Row has null join key", nameof(row));
-
       /**************************************TEST METHOD(S)*********************************************/
       /// <summary>
       /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -204,7 +173,8 @@ namespace BCG
       /// </summary>
       /// <param name="leftPacket"></param>
       /// <param name="rightPacket"></param>
-      public void test(IDataPacket dp, bool isLeft) {
+      public void test (DataPacket dp, bool isLeft)
+      {
          JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
          jo.changeHandler (dp);
       }

src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs

@@ -1,6 +1,5 @@
-using Spark.Core.Diagnostics;
-
-using System.Collections.Generic;
+using System;
+using System.Collections;
 
 namespace BCG
 {
@@ -9,7 +8,7 @@ namespace BCG
 	/// </summary>
    internal class JoinDeletionIndex
    {
-      Dictionary<object, RowList> myIndex = new();
+      IDictionary myIndex = new Hashtable ();
       private string myOpIdName ;
       private bool myIsLeft;
 
@@ -29,8 +28,10 @@ namespace BCG
       /// <param name="row">JoinRow to add</param>
       public void add (JoinRow row)
       {
-         object o = ExceptionAssert.NotNull(row[myOpIdName]);
-         RowNode nodeAdded = myIndex.GetOrAdd(o, _ => new()).add(row);
+         object o = row[myOpIdName];
+         if (myIndex[o] == null)
+            myIndex[o] = new RowList ();
+         RowNode nodeAdded = ((RowList)myIndex[o]).add (row);
          if (myIsLeft)
             row.leftNode = nodeAdded;
          else
@@ -42,10 +43,7 @@ namespace BCG
       /// To remove a node, if node does not exist exception is thrown
       /// </summary>
       /// <param name="node"></param>
-      public void remove (RowNode node) {
-         var nodeId = ExceptionAssert.NotNull(node.row?[myOpIdName]);
-         myIndex[nodeId].remove(node);
-      }
+      public void remove (RowNode node)   {((RowList)myIndex[node.row[myOpIdName]]).remove (node);}
 
       public RowList getMatchingRows (object key)  {
          RowList retVal = (RowList)myIndex[key];

src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs

@@ -1,6 +1,7 @@
-using bts;
+using System;
+using BCG.Utilities;
 
-using Bts.DataStreamCore;
+using bts;
 
 namespace BCG
 {
@@ -8,11 +9,11 @@ namespace BCG
 	{
 		public JoinForeignTable (JoinTableSchema schema) : base (schema) {}
 
-      public override void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
-         var newRow = new JoinRow(leftRow, rightRow, mySchema);
-         int leftId = leftRow.Id();
-         Log.assert(myIdIndex[leftId] == null);
-         myIdIndex[leftId] = newRow;
+      public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
+      {
+         JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema);
+         Log.assert(myIdIndex[leftRow["id"]] == null);
+         myIdIndex[leftRow["id"]] = newRow;
          ((BlueTable)(dp.addedRows)).addRow (newRow);
          myLeftIndex.add (newRow);
          myRightIndex.add (newRow);

src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs

@@ -1,12 +1,11 @@
+using System;
+using System.Collections;
+using BCG.Utilities;
+
 using bts;
 
 using Bts.ColumnMapExpressions;
 
-using Spark.Core.Diagnostics;
-
-using System;
-using System.Collections;
-
 namespace BCG
 {
 	/// <summary>
@@ -38,8 +37,7 @@ namespace BCG
       private JoinTableSchema myJoinSchema;
       private IBlueTableSchema myLeftSchema, myRightSchema;
       private string myLeftColExpression, myRightColExpression;
-      string[] myUserRules;
-      string[]? myResolvedNames;
+      private string[] myUserRules, myResolvedNames;
       private IDictionary myFinalMap = new Hashtable ();
 
 
@@ -89,18 +87,16 @@ namespace BCG
       /// </summary>
       private void myFormUserRules ()
       {
-         var leftExp = ColumnMapExpression.Parse(myLeftColExpression) ?? throw new InvalidOperationException("Failed to parse left column expression");
-         var rightExp = ColumnMapExpression.Parse(myRightColExpression) ?? throw new InvalidOperationException("Failed to parse right column expression");
          if (DataStreamLoadFileConfig.instance.useStaticParser)
          {
-            myFormUserRules(true, leftExp);
-            myFormUserRules(false, rightExp);
+            myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
+            myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
          }
          else
          {
             // TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
-            myFormUserRules(true, leftExp);
-            myFormUserRules(false, rightExp);
+            myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
+            myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
          }
 
       }
@@ -119,7 +115,13 @@ namespace BCG
                OriginalNameInfo oni = myOriginalColInfo[i];
                if (oni.isLeft == isLeft)
                {
-                  myUserRules[i] = userMap[oni.colName] as string ?? oni.colName;
+                  if (userMap[oni.colName] != null)
+                  {
+                     if ((string)userMap[oni.colName] != oni.colName)
+                        myUserRules[i] = (string)userMap[oni.colName];
+                     else
+                        myUserRules[i] = oni.colName;
+                  }
                }
             }
          else
@@ -129,13 +131,10 @@ namespace BCG
                OriginalNameInfo oni = myOriginalColInfo[i];
                if (oni.isLeft == isLeft)
                {
-                  var colName = userMap[oni.colName] as string;
-                  // myUserRules[i] = userMap[oni.colName] as string 
-                  if (colName is null) {
+                  if (userMap[oni.colName] == null)
                      myUserRules[i] = oni.colName;
-                  } else if (colName != oni.colName) {
-                     myUserRules[i] = colName;
-                  }
+                  else if ((string)userMap[oni.colName] != oni.colName)
+                     myUserRules[i] = (string)userMap[oni.colName];
                }
             }
          }
@@ -166,12 +165,26 @@ namespace BCG
 
       private void myFormFinalMap ()
       {
-         ExceptionAssert.NotNull(myResolvedNames);
          foreach (string resolvedName in myResolvedNames)
             if (resolvedName != null)
                myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)];
       }
 
+
+      /// <summary>
+      /// Utility method to invert a hashtable
+      /// </summary>
+      /// <param name="arg"></param>
+      /// <returns></returns>
+   
+      private IDictionary reciprocateDictionary (IDictionary arg)
+      {
+         IDictionary retVal = new Hashtable ();
+         foreach (DictionaryEntry de in arg)
+            retVal[de.Value] = de.Key;
+         return retVal;
+      }
+
       /// <summary>
       /// Utility method to find all occurances of an obkect in an IList
       /// </summary>
@@ -181,11 +194,9 @@ namespace BCG
       private ArrayList myFindAllInCollection (IList list, object obj)
       {
          ArrayList retVal = new ArrayList ();
-         for (int i = 0; i < list.Count; i++) {
-            if (Equals(list[i], obj)) {
+         for (int i = 0; i < list.Count; i++)
+            if (list[i] != null && list[i].Equals (obj))
                retVal.Add (i);
-            }
-         }
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs

@@ -1,7 +1,6 @@
-using Spark.Core.Diagnostics;
-
 using System;
-using System.Diagnostics.CodeAnalysis;
+using System.Collections;
+using BCG.Utilities;
 
 namespace BCG
 {
@@ -12,46 +11,34 @@ namespace BCG
    public class JoinOperand 
    {
       private string myJoinCol;
-      IDataPacket? myLastArrivedPacket;
-      JoinOperand? myOtherOperand;
+      private DataPacket myLastArrivedPacket;
+      private JoinOperand myOtherOperand;
       private bool myIsLeft;
-      JoinOperandRowIndex? myRowIndex;
-      InternalJoinStream? myMaster;
-
+      private JoinOperandRowIndex myRowIndex;
+      private InternalJoinStream myMaster;
+      private IBlueTableSchema myOpSchema;
+      
       public JoinOperand (string joinCol, bool isLeft)
       {
          myJoinCol = joinCol;
          myIsLeft = isLeft;
       }
 
-      [MemberNotNullWhen(true, nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
-      bool isInitialized {
-         get {
-            if (myLastArrivedPacket is null) {
-               return false;
-            }
-            ExceptionAssert.NotNull(myMaster, myLastArrivedPacket, operandSchema, myOtherOperand, myRowIndex);
-            return true;
-         }
-      }
-
       internal void changeHandler (object o)
       {
-         var dp = (IDataPacket)o;
-         if (isInitialized) {
-            mySubsequentChangeHandler (dp);
-         } else {
-            myFirstChangeHandler(dp);
-         }
+         DataPacket temp = (DataPacket)o;
+         if (myLastArrivedPacket != null)
+            mySubsequentChangeHandler ((DataPacket)o);
+         else
+            myFirstChangeHandler ((DataPacket)o);
       }
 
-      [MemberNotNull(nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
-      bool myFirstChangeHandler(IDataPacket arrivedDp) {
-         ExceptionAssert.NotNull(myMaster);
+      private void myFirstChangeHandler (DataPacket arrivedDp)
+      {
          myLastArrivedPacket = arrivedDp;
-         operandSchema = myLastArrivedPacket.newRows.schema;
+         myOpSchema = myLastArrivedPacket.newRows.schema;
          myOtherOperand = myMaster.getOtherOperand (this);
-         myRowIndex = new JoinOperandRowIndex (operandSchema[myJoinCol].colNo, operandSchema["id"].colNo);
+         myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo);
          if (myOtherOperand.hasReceivedData)
          {
             //the other operand has already received data, so the join would
@@ -62,16 +49,14 @@ namespace BCG
             myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft);
             myMaster.endSession ();
          }
-         else {
+         else
             myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows, 
-               operandSchema[myJoinCol].colNo,
-               operandSchema["id"].colNo);
-         }
-         return true;
+               myOpSchema[myJoinCol].colNo,
+               myOpSchema["id"].colNo);
       }
 
-      void mySubsequentChangeHandler (IDataPacket arrivedDp) {
-         RequireInitialized();
+      private void mySubsequentChangeHandler (DataPacket arrivedDp)
+      {
          myLastArrivedPacket = arrivedDp;
          if (!myOtherOperand.hasReceivedData)
          {
@@ -100,46 +85,29 @@ namespace BCG
          myMaster.endSession ();
       }
 
-      internal void addRow (IBlueRow row) =>
-         ExceptionAssert.NotNull(myRowIndex).add(row);
+      internal void addRow (IBlueRow row) {myRowIndex.add (row);}
 
-      internal void deleteRow (IBlueRow row) {
-         RequireInitialized();
-         myRowIndex.remove (row);
-      }
+      internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);}
 
-      internal IBlueTableSchema? operandSchema { get; private set; }
+      internal IBlueTableSchema operandSchema  {get{return myOpSchema;}}
 
       internal InternalJoinStream master {set{myMaster = value;}}
 
       internal string joinCol   {get{return myJoinCol;}}
 
-      internal int joinColOridinal {
-         get {
-            RequireInitialized();
-            return operandSchema[myJoinCol].colNo;
-         }
-      }
+      internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}}
   
-      internal RowList getMatchingRows (object key) {
-         RequireInitialized();
-         return myRowIndex.getMatchingRows(key);
-      }
+      internal RowList getMatchingRows (object key)   {return (RowList) myRowIndex.getMatchingRows (key);}
 
       internal bool hasReceivedData   {get{return myLastArrivedPacket != null;}}
 
-      internal void Reset() {
+      internal void init ()  
+      {
          ///should be brought back to the the state before the first callback 
          ///(ie the state after construction)
          myRowIndex = null;
          myLastArrivedPacket = null;
       }
-
-      [MemberNotNull(nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
-      internal void RequireInitialized() {
-         if (!isInitialized) {
-            throw new InvalidOperationException("Internal state not properly initialized");
-         }
-      }
+      
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs

@@ -1,6 +1,6 @@
-using Spark.Core.Diagnostics;
-
-using System.Collections.Generic;
+using System;
+using System.Collections;
+using BCG.Utilities;
 
 namespace BCG
 {
@@ -11,10 +11,8 @@ namespace BCG
 	/// </summary>
 	internal class JoinOperandRowIndex
 	{
-      readonly Dictionary<object, RowList> myJoinColDict = new();
-      readonly Dictionary<int, RowNode> myIdDict = new();
-      readonly int myJoinColOrdinal;
-      readonly int myIdColOrdinal;
+      private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
+      private int myJoinColOrdinal, myIdColOrdinal;
 
       /// <summary>
       /// 
@@ -36,14 +34,19 @@ namespace BCG
 
       public void add (IBlueRow row)
       {
-         myIdDict[getId(row)] = myJoinColDict.GetOrAdd(getJoinKey(row), _ => new()).add(row);
+         RowList list;
+         if (myJoinColDict[row[myJoinColOrdinal]] == null)
+            myJoinColDict[row[myJoinColOrdinal]] = list = new RowList (); 
+         else
+            list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
+         myIdDict[row[myIdColOrdinal]] = list.add (row);
       }
 
       public void remove (IBlueRow row)
       {
-         var id = getId(row);
-         myJoinColDict[getJoinKey(row)].remove(myIdDict[id]);
-         myIdDict.Remove(id);
+         RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
+         ((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
+         myIdDict.Remove (row[myIdColOrdinal]);
       }
 
       public RowList getMatchingRows (object key)  
@@ -59,11 +62,5 @@ namespace BCG
          myJoinColDict.Clear ();
          myIdDict.Clear ();
       }
-
-      object getJoinKey(IBlueRow row) =>
-         ExceptionAssert.NotNull(row[myJoinColOrdinal]);
-
-      int getId(IBlueRow row) =>
-         (int)row[myIdColOrdinal]!;
 	}
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs

@@ -1,21 +1,17 @@
-using bts;
-
-using Bts.DataStreamCore;
-
 using System;
+using BCG.Utilities;
 using System.Collections;
+using bts;
 
 namespace BCG
 {
    internal class JoinRow : IBlueRow 
    {
-      RowNode? myLeftNode;
-      RowNode? myRightNode;
-      protected IBlueRow myLeftRow;
-      protected IBlueRow? myRightRow;
+      private RowNode myLeftNode, myRightNode;
+      protected IBlueRow myLeftRow, myRightRow;
       protected int myId;
       protected JoinTableSchema mySchema;
-      private object?[] myValues;
+      private object[] myValues;
       
       #region Constructor(s)
 
@@ -25,8 +21,8 @@ namespace BCG
       /// <param name="leftRow">left row of the join</param>
       /// <param name="rightRow">right row of the join</param>
       /// <param name="schema">schema of the joined row</param>
-      public JoinRow (IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) 
-         : this(leftRow.Id(), leftRow, rightRow, schema) { }
+      public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) 
+         : this ((int)leftRow["id"], leftRow, rightRow, schema)   {}
 
       /// <summary>
       /// Constructs a row assigning it the ID provioded as the first argument
@@ -35,7 +31,8 @@ namespace BCG
       /// <param name="leftRow">left row of the join</param>
       /// <param name="rightRow">right row of the join</param>
       /// <param name="schema">schema of the joined row</param>
-      public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) {
+      public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
+      {
          myId = id;
          myLeftRow = leftRow;
          myRightRow = rightRow;
@@ -51,13 +48,13 @@ namespace BCG
 
       public bool isPartialJoin{get{return myRightRow == null;}}
 
-      public RowNode? leftNode 
+      public RowNode leftNode 
       {
          get{return myLeftNode;}
          set{myLeftNode = value;}
       }
 
-      public RowNode? rightNode
+      public RowNode rightNode
       {
          get{return myRightNode;}
          set{myRightNode = value;}
@@ -88,7 +85,7 @@ namespace BCG
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public virtual object? this[object key]
+      public virtual object this [object key]
       {
          get
          {
@@ -99,7 +96,7 @@ namespace BCG
          set {throw new NotSupportedException ("The IBlueRow is read only");}
       }
 
-      public object? this[int index]
+      public object this [int index]
       {
          get
          {
@@ -122,8 +119,7 @@ namespace BCG
          }
       }
 
-      public object? this[string colName] =>
-         this[mySchema[colName].colNo];
+      public object this [string colName] {get{return this[mySchema[colName].colNo];}}
    
       public ICollection Keys{get{return mySchema.columnNames;}}
 
@@ -137,8 +133,7 @@ namespace BCG
          }
       }
 
-      public void Add (object key, object? value) =>
-         throw new NotSupportedException ("The IBlueRow is readonly");
+      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
 
       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs

@@ -1,26 +1,20 @@
-using bts;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-using Spark.Core.Disposables;
-
 using System;
+using System.Data;
 using System.Collections;
+using System.Text.RegularExpressions;
+using BCG.Utilities;
+using BCG.Utilities.General;
+using bts;
 
 namespace BCG
 {
    public class JoinStream : ViewStream
    {
-      protected readonly InternalJoinStream? myIjs;
-      protected readonly ColumnMap? myLeftCm;
-      protected readonly ColumnMap? myRightCm;
-      protected readonly bool myIsProject = true;
+      protected InternalJoinStream? myIjs;
       protected ProjectStream? myPs;
-
-      readonly bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
-      IDisposable? internalSubscription;
-      IDisposable? projectSubscription;
+      protected ColumnMap? myLeftCm, myRightCm;
+      protected bool myIsProject = true;
+      private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
       
       protected JoinStream () {}
       public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1,
@@ -117,61 +111,47 @@ namespace BCG
          {
             if (myPs == null)
             {
-               ExceptionAssert.NotNull(myIjs, myLeftCm, myRightCm);
-               var leftSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Left schema is null");
-               var rightSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Right schema is null");
-               var joinSchema = (JoinTableSchema)myIjs.schema;
-               JoinNameResolver resolver = new JoinNameResolver(
-                  joinSchema, leftSchema, myLeftCm.columnExpression, rightSchema, myRightCm.columnExpression);
+               JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression, 
+                  myIjs.getSchema (false), myRightCm.columnExpression);
                IDictionary  map = resolver.getMap ();
                map["id"] = "id";
-               Disposable.Dispose(ref projectSubscription);
                myPs = new ProjectStream (myIjs, map);
-               projectSubscription = myPs.Subscribe(myProjectChangeHandler);
+               myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
                return;
             }
             else
                Log.assert(false, "This call back should not have been called");
          }
-         else {
-            myFireEvent(cloneDataPacket(o));
-         }
+         else
+            myFireEvent ((o as ICloneable).Clone());
       }
 
       private void myProjectChangeHandler (object o)
       {
-         ExceptionAssert.NotNull(myIjs);
-         if (myIjs.numberOfSubscribers > 1) {
-            Disposable.Dispose(ref internalSubscription);
-         }
-         myFireEvent(cloneDataPacket(o));
+         if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
+            myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myFireEvent ((o as ICloneable).Clone());
+         
       }
 
       protected override void myOnFirstSubscriber ()
       {
-         Log.isnull(internalSubscription);
-         internalSubscription = ExceptionAssert.NotNull(myIjs).Subscribe(myInternalJoinChangeHandler);
+         myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
       }
 
       protected override void myOnLastSubscriber ()
       {
-         Disposable.Dispose(ref internalSubscription);
+         // how does this relate to the -= in myProjectChangeHandler?
+         myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+
          // if there is a  project after the ijs we need to 
          // unsubscribe to it otherwise references will be kept and 
          // a memory leak will occur
-         Disposable.Dispose(ref projectSubscription);
+         if (myPs != null)
+            myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
          myPs = null;
       }
 
-      object cloneDataPacket(object dpObj) {
-         var rv = (dpObj as ICloneable)?.Clone();
-         if (rv is null) {
-            var dp = (IDataPacket)dpObj;
-            rv = new DataPacket(new BlueTable(dp.newRows), new BlueTable(dp.addedRows), new BlueTable(dp.deletedRows));
-         }
-         return rv;
-      }
-
       /**************************************TEST METHOD(S)*********************************************/
       /// <summary>
       /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -179,9 +159,7 @@ namespace BCG
       /// </summary>
       /// <param name="leftPacket"></param>
       /// <param name="rightPacket"></param>
-      public void test (DataPacket dp, bool isLeft) {
-         ExceptionAssert.NotNull(myIjs).test(dp, isLeft);
-      }
+      public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);}
       /***********************************************************************************************/
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs

@@ -1,17 +1,14 @@
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
-using System.Collections.Generic;
+using System.IO;
+using BCG.Utilities.General;
 
 namespace BCG
 {
    internal class JoinTable : IBlueRowCollection
    {
       protected JoinTableSchema mySchema;
-      protected readonly Dictionary<int, IBlueRow> myIdIndex = new();
+      protected IDictionary myIdIndex = new Hashtable ();
       protected JoinDeletionIndex myLeftIndex, myRightIndex;
       private int myNewRowId;
 
@@ -22,16 +19,19 @@ namespace BCG
          myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
       }
 
-      public IBlueRow this [int id] {
-         get {
-            var rowObj = myIdIndex[id]
-               ?? throw new BlueDataException($"ID {id} does not exist in table {mySchema.tableName}");
-            return (IBlueRow)rowObj;
+      public IBlueRow this [int id] 
+      {
+         get
+         {
+            if (myIdIndex[id] == null)
+               throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
+            return (IBlueRow)myIdIndex[id];
          }
       }
 
       #region JoinTable members
-      public virtual void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
+      public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
+      {
          JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
          myIdIndex[myNewRowId ++] = newRow;
          ((BlueTable)(dp.addedRows)).addRow (newRow);
@@ -40,7 +40,7 @@ namespace BCG
             myRightIndex.add (newRow);
       }
 
-      public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, IDataPacket dp)
+      public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, DataPacket dp)
       {
          JoinDeletionIndex thisIndex = myRightIndex, otherIndex = myLeftIndex;
          if (isLeft) 
@@ -48,17 +48,17 @@ namespace BCG
             thisIndex = myLeftIndex;
             otherIndex = myRightIndex;
          }
-         RowList matchingNodes = thisIndex.getMatchingRows(deletedRow.Id());
+         RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]);
          foreach (RowNode node in matchingNodes)
          {
-            var row = ExceptionAssert.Argument.Is<JoinRow>(node.row);
-            RowNode? correspondingNode = isLeft ? row.rightNode : row.leftNode;
+            JoinRow row = (JoinRow)node.row;
+            RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode;
             if (correspondingNode != null)
                otherIndex.remove (correspondingNode); //in case of partial left joinss
-            myIdIndex.Remove(row.Id());
+            myIdIndex.Remove (row["id"]);
             ((BlueTable)dp.deletedRows).addRow (row);
          }
-         thisIndex.removeAll(deletedRow.Id());
+         thisIndex.removeAll (deletedRow["id"]);
       }
       #endregion
       #region ICollection members
@@ -68,11 +68,7 @@ namespace BCG
 
       public object SyncRoot{get{return this;}}
 
-      public void CopyTo (Array array, int index) {
-         foreach (var row in myIdIndex.Values) {
-            array.SetValue(row, index++);
-         }
-      }
+      public void CopyTo (Array array, int index)  {myIdIndex.Values.CopyTo (array, index);}
       #endregion
 
       #region IEnumerable members

src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs

@@ -1,7 +1,8 @@
-using bts;
+using System;
+using System.Collections;
+using BCG.Utilities;
 
-using System.Collections.Generic;
-using System.Linq;
+using bts;
 
 namespace BCG
 {
@@ -12,11 +13,10 @@ namespace BCG
    public class JoinTableSchema : BlueTableSchema
    {
       private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
-      readonly Dictionary<string, string> myLeftNameToAlias = new();
-      readonly Dictionary<string, string> myRightNameToAlias = new();
-      readonly Dictionary<string, string> myAliasToLeftName = new();
-      readonly Dictionary<string, string> myAliasToRightName = new();
-
+      private IDictionary myLeftNameToAlias = new Hashtable ();
+      private IDictionary myRightNameToAlias = new Hashtable ();
+      private IDictionary myAliasToLeftName = new Hashtable(),
+                           myAliasToRightName = new Hashtable();
       /// <summary>
       /// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
       /// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
@@ -85,9 +85,9 @@ namespace BCG
             renamed = false;
          }
          myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
-         myIndexByColOrder[myIndexByColOrder.Length - 1] = myIndexByColName["id"];
-         myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
-         myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
+         myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"];
+         myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias);
+         myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias);
       }
 
       public string leftJoinCol  {get{return myLeftJc;}}

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,21 +1,17 @@
 using bts;
 using bts.collections;
 
-using Bts.DataStreamCore;
-
 using mbts;
 
-using Spark.Core.Diagnostics;
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Security.Cryptography;
 
 namespace BCG {
    public class NonIncJoinBookkeeper : IDisposable {
-      public event Action? on_new_data;
+      public Action on_new_data;
       private readonly NonIncJoinSpec _join_spec;
       private readonly int _join_position;
       private readonly int _start_index_in_output;
@@ -24,14 +20,13 @@ namespace BCG {
       private readonly bool _is_tag_along;
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
-      IBlueTableSchema? _schema;
+      private IBlueTableSchema _schema = null;
       private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
       private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
-      IBlueRow? _solitary_row;
+      private IBlueRow _solitary_row = null;
 
-      int[]? _join_col_indices;
-      int[]? _data_col_indices;
-      IDisposable? subscription;
+      private int[] _join_col_indices = null,
+                     _data_col_indices = null;
 
       public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
          _join_spec = the_join_spec;
@@ -43,7 +38,7 @@ namespace BCG {
       }
 
       private bool _filled_missing_data = false;
-      void _fill_missing_data(IDataPacket dp) {
+      private void _fill_missing_data(DataPacket dp) {
          if (_filled_missing_data)
             return;
          Func<string[], bool> null_or_empty = (str_arr) => {
@@ -57,17 +52,17 @@ namespace BCG {
             return true;
          };
          Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.Schema.columnNames);
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
             List<string> cols_list = new List<string>(col_names);
             cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
             return cols_list.ToArray();
          };
          if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.Schema.columnNames;
+            _join_spec.data_col_names = dp.schema.columnNames;
          }
          _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
          if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.Schema.columnNames;
+            _join_spec.data_col_names = dp.schema.columnNames;
          }
          if (null_or_empty(_join_spec.projected_col_names)) {
             _join_spec.projected_col_names = _join_spec.data_col_names;
@@ -75,16 +70,20 @@ namespace BCG {
       }
 
       public void init() {
-         ExceptionAssert.MemberIsNull(subscription);
-         subscription = ExceptionAssert.MemberNotNull(_join_spec.stream).Subscribe(_on_stream_cb);
+         _join_spec.stream.onChange += _on_stream_cb;
       }
 
       public void _on_stream_cb(object dp_obj) {
          //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
-         var dp = (IDataPacket)dp_obj;
+         DataPacket dp = (DataPacket)dp_obj;
          _fill_missing_data(dp);
-         init(dp);
+         if (_schema == null) {
+            _schema = dp.schema;
+            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
+            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
+         }
+
          if (_is_left_most) {
             _row_list.Clear();
             foreach (IBlueRow r in dp.newRows)
@@ -109,68 +108,44 @@ namespace BCG {
             on_new_data();
       }
 
-      [MemberNotNullWhen(true, nameof(_schema))]
       public bool is_ready { get { return _schema != null; } }
 
       public void constribute_cols_to_schema(List<ColumnInfo> cis) {
-         requireReady();
          for (int i = 0; i < _join_spec.data_col_names.Length; i++)
             cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
       }
 
       public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
 
-      public void copy_master_row_to_output_row(int row_index, object?[] output_row) =>
-         copyRow(ExceptionAssert.NotNull(_row_list[row_index].second), output_row);
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+         IBlueRow this_row = _row_list[row_index].second;
+         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+      }
 
-      public void copy_matching_row_to_output_row(int row_hash, object?[] output_row) {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
          if (_is_tag_along) {
             if (_solitary_row != null)
-               copyRow(_solitary_row, output_row);
+               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
             else
                copy_dbnull_to_output_row(output_row);
          } else {
-            if (_row_map.TryGetValue(row_hash, out var this_row))
-               copyRow(this_row, output_row);
+            IBlueRow this_row = null;
+            if (_row_map.TryGetValue(row_hash, out this_row))
+               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
             else
                copy_dbnull_to_output_row(output_row);
          }
       }
 
-      void copy_dbnull_to_output_row(object?[] output_row) {
-         requireDataColIndices();
+      private void copy_dbnull_to_output_row(object[] output_row) {
          for (int i = 0; i < _data_col_indices.Length; i++)
             output_row[i + _start_index_in_output] = DBNull.Value;
       }
 
       #region IDisposable
 
-      public void Dispose() =>
-         Disposable.Dispose(ref subscription);
+      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
 
       #endregion
-
-      void copyRow(IBlueRow src, object?[] dst) {
-         requireDataColIndices();
-         BlueDataUtility.copy_to_row(src, _data_col_indices, dst, _start_index_in_output);
-      }
-
-      [MemberNotNull(nameof(_schema), nameof(_join_col_indices), nameof(_data_col_indices))]
-      void init(IDataPacket dp) {
-         _schema ??= dp.Schema;
-         _join_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
-         _data_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
-      }
-
-      [MemberNotNull(nameof(_data_col_indices))]
-      int[] requireDataColIndices() =>
-         ExceptionAssert.NotNull(_data_col_indices);
-
-      [MemberNotNull(nameof(_schema))]
-      void requireReady() {
-         if (!is_ready) {
-            throw new InvalidOperationException("Not ready");
-         }
-      }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -1,25 +1,22 @@
-using mbts;
-
-using System;
+using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using mbts;
 
 namespace BCG {
    // this a collection of data, theres no joining logic in here
    public class NonIncJoinSpec {
-      public ViewStream? stream;
+      public ViewStream stream;
       // join col names are the names on which the streams are joined (for equality),
       // data col names are the names in the original stream
       // projected col names are their new names, ie if its x:l_x, then x is in the 
       // data_col_names and l_x in projected cols
-      public string[] join_col_names;
-      public string[] data_col_names;
-      public string[] projected_col_names;
+      public string[] join_col_names, data_col_names, projected_col_names;
 
-      public NonIncJoinSpec() {
-         join_col_names = Array.Empty<string>();
-         data_col_names = Array.Empty<string>();
-         projected_col_names = Array.Empty<string>();
-      }
+      public NonIncJoinSpec() { }
 
       public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
          : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -5,19 +5,18 @@ using mbts;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
 
 namespace BCG {
    [DataStreamUserRespresentation("fastJoin")]
    public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
       private readonly NonIncJoinSpec[] _join_specs;
-      readonly INonIncJoinInlineMapper? _output_mapper;
+      private readonly INonIncJoinInlineMapper _output_mapper;
       private readonly BlueRowHasher _row_hasher;
       private readonly TimeSpan _throttle_period;
       private readonly ViewStreamDatabase _vsd;
 
       private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
-      IBlueTableSchema? _output_schema;
+      private IBlueTableSchema _output_schema = null;
       private DateTime _last_fire_time = DateTime.MinValue;
       private bool _is_dirty = false;
 
@@ -36,10 +35,10 @@ namespace BCG {
                               params Triplet<ViewStream, string, string>[] join_specs)
          : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
 
-      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0)
+      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
          : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
 
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0) {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
          _join_specs = join_specs;
          _output_mapper = output_mapper;
          _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -89,7 +88,6 @@ namespace BCG {
          return rv;
       }
 
-      [MemberNotNull(nameof(_output_schema))]
       private void _construct_schema() {
          if (_output_schema == null) {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });

src/DataStreamSuite/DataStream/Primitives/Join/ParallelJoinTest.cs

@@ -1,7 +1,9 @@
-#nullable disable
-using System;
+using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Text;
+using BCG.Utilities.General;
+using BCG.Utilities;
 
 namespace BCG
 {

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs

@@ -2,12 +2,11 @@ namespace bts {
    using BCG;
    using BCG.Utilities.General;
 
-   using Spark.Core.Diagnostics;
-   using Spark.Core.Disposables;
-
    using System;
    using System.Diagnostics;
 
+   using ZstdSharp.Unsafe;
+
    // functon that joins the 2 rows and returns the outputput row. The function
    // can return a null which means the 2 rows dont need to be joined
    using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
@@ -17,10 +16,6 @@ namespace bts {
       private bool _dirty = false;
       private Stopwatch _stopwatch = new Stopwatch();
       private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
-      IDisposable? subscription;
-      IBlueRowCollection? leftRows;
-      IBlueRowCollection? rightRows;
-
       public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
          _lvs = lvs;
          _rvs = rvs;
@@ -28,56 +23,46 @@ namespace bts {
       }
 
       protected override void myOnFirstSubscriber() {
-         _lvs.onChange += OnLeftChange;
-         _rvs.onChange += OnRightChange;
-         subscription = Disposable.Create(Unsubscribe);
+         _lvs.onChange += _handle_data;
+         _rvs.onChange += _handle_data;
          _vsd.addWorker(this);
-
-         void OnLeftChange(object dp_obj) {
-            leftRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
-            _dirty = true;
-            doWork();
-         }
-
-         void OnRightChange(object dp_obj) {
-            rightRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
-            _dirty = true;
-            doWork();
-         }
-
-         void Unsubscribe() {
-            _lvs.onChange -= OnLeftChange;
-            _rvs.onChange -= OnRightChange;
-         }
       }
 
       protected override void myOnLastSubscriber() {
-         subscription?.Dispose();
-         subscription = null;
+         _lvs.onChange -= _handle_data;
+         _rvs.onChange -= _handle_data;
          _vsd.removeWorker(this);
       }
 
+      private void _handle_data(object dp_obj) {
+         _dirty = true;
+         doWork();
+      }
+
       public void doWork() {
-         if (!_dirty || leftRows is null || rightRows is null) {
-            return;
-         }
+         Func<bool> all_data_revcd = () => {
+            return _lvs.rows != null && _rvs.rows != null;
+         };
          _stopwatch.Start();
-         _dirty = false;
-         BlueTable? rv = null;
-         foreach (IBlueRow lr in leftRows) {
-            foreach (IBlueRow rr in rightRows) {
-               IBlueRow resr = _join_func(lr, rr);
-               if (resr == null)
-                  continue;
-               rv = rv ?? new BlueTable(resr.schema);
-               rv.addRow(resr);
+         if (_dirty && all_data_revcd()) {
+            _dirty = false;
+            BlueTable rv = null;
+            foreach (IBlueRow lr in _lvs.rows) {
+               foreach (IBlueRow rr in _rvs.rows) {
+                  IBlueRow resr = _join_func(lr, rr);
+                  if (resr == null)
+                     continue;
+                  rv = rv ?? new BlueTable(resr.schema);
+                  rv.addRow(resr);
+               }
             }
+            _stopwatch.Stop();
+            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
+            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+            myFireEvent(dp);
          }
-         _stopwatch.Stop();
-         Log.Gen.info($"ProximityJoinStream left_count: {leftRows.Count} right_count: {rightRows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs");
-         var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
-         myFireEvent(dp);
+
       }
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs

@@ -1,5 +1,3 @@
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
 
@@ -11,7 +9,7 @@ namespace BCG
    /// </summary>
    internal class RowList : ICollection
    {
-      RowNode? myFirstPtr;
+      private RowNode myFirstPtr;
       private int myCount = 0;
      
       #region ICollection methods
@@ -69,19 +67,19 @@ namespace BCG
          }
          else if (node.isFirst)
          {
-            myFirstPtr = ExceptionAssert.NotNull(myFirstPtr).next;
-            ExceptionAssert.NotNull(myFirstPtr).previous = null;
+            myFirstPtr = myFirstPtr.next;
+            myFirstPtr.previous = null;
             return;
          }
          else if (node.isLast)
          {
-            ExceptionAssert.NotNull(node.previous).next = null;
+            node.previous.next = null;
             return;
          }
          else if (!node.isFirst && !node.isLast)
          {
-            ExceptionAssert.NotNull(node.previous).next = node.next;
-            ExceptionAssert.NotNull(node.next).previous = node.previous;
+            node.previous.next = node.next;
+            node.next.previous = node.previous;
             return;
          }
          else
@@ -93,16 +91,14 @@ namespace BCG
 
       internal class RowListEnumerator : IEnumerator
       {
-         RowNode? myPointer;
+         private RowNode myPointer;
          private RowList myList;
          private int myPos = -1;
          public RowListEnumerator (RowList list)   {myList = list;}
-         public object? Current => myPointer;
+         public object Current   {get{return myPointer;}}
          public bool MoveNext()
          {
-            myPointer = myPos == -1
-               ? myList.myFirstPtr
-               : ExceptionAssert.NotNull(myPointer).next;
+            myPointer = myPos == -1 ? myList.myFirstPtr : myPointer.next;
             myPos ++;
             return myPointer != null;  
          }
@@ -112,24 +108,28 @@ namespace BCG
 
    internal class RowNode
    {
-      public RowNode (IBlueRow row) =>
-         this.row = row;
-
-      public RowNode (IBlueRow row, RowList list)
-         : this(row) =>
-         this.list = list;
-
-      public bool isLast => next is null;
-      public bool isFirst => previous is null;
-      public RowList? list { get; }
-      public IBlueRow? row { get; set; }
-      public RowNode? next { get; set; }
-      public RowNode? previous { get; set; }
-
-      public RowList RequireList() =>
-         ExceptionAssert.MemberNotNull(list);
-
-      public IBlueRow RequireRow() =>
-         ExceptionAssert.MemberNotNull(row);
+      private IBlueRow myRow;
+      private RowNode myNext;
+      private RowNode myPrevious;
+      private RowList myList;
+      
+      public RowNode (IBlueRow row){myRow = row;}
+      public RowNode (IBlueRow row, RowList list) : this (row)  {myList = list;}
+      public bool isLast {get{return myNext == null;}}
+      public bool isFirst {get{return myPrevious == null;}}
+      public RowList list {get{return myList;}}
+      public IBlueRow row
+      {
+         get{return myRow;}
+         set{myRow = value;}
+      }
+      public RowNode next {
+         get{return myNext;}
+         set{myNext = value;}
+      }
+      public RowNode previous  {
+         get{return myPrevious;}
+         set{myPrevious = value;}
+      }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs

@@ -1,17 +1,125 @@
-using Bts.DataStreamCore;
-
-using mbts;
-
-using Spark.Core.Collections;
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
+using mbts;
+using Spark.Core.Collections;
 
 namespace BCG
 {
+   internal class SingleRowJoinDicionary<TKey, TValue> : IDictionary<TKey, TValue>
+   {
+      private TKey myKey;
+      private TValue myValue;
+
+      #region IDictionary<TKey,TValue> Members
+
+      public void Add(TKey key, TValue value)
+      {
+         myKey = key;
+         myValue = value;
+      }
+
+      public bool ContainsKey(TKey key)
+      {
+         return myKey.Equals(key);
+      }
+
+      public ICollection<TKey> Keys
+      {
+         get { return new TKey[] { myKey }; }
+      }
+
+      public bool Remove(TKey key)
+      {
+         bool rv = myKey.Equals(key);
+         myKey = default(TKey);
+         return rv;
+      }
+
+      public bool TryGetValue(TKey key, out TValue value)
+      {
+         bool rv = false;
+         value = (rv = myKey.Equals(key)) ? myValue : default(TValue);
+         return rv;
+      }
+
+      public ICollection<TValue> Values
+      {
+         get { throw new Exception("The method or operation is not implemented."); }
+      }
+
+      public TValue this[TKey key]
+      {
+         get
+         {
+            throw new Exception("The method or operation is not implemented.");
+         }
+         set
+         {
+            throw new Exception("The method or operation is not implemented.");
+         }
+      }
+
+      #endregion
+
+      #region ICollection<KeyValuePair<TKey,TValue>> Members
+
+      public void Add(KeyValuePair<TKey, TValue> item)
+      {
+         throw new Exception("The method or operation is not implemented.");
+      }
+
+      public void Clear()
+      {
+         throw new Exception("The method or operation is not implemented.");
+      }
+
+      public bool Contains(KeyValuePair<TKey, TValue> item)
+      {
+         throw new Exception("The method or operation is not implemented.");
+      }
+
+      public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
+      {
+         throw new Exception("The method or operation is not implemented.");
+      }
+
+      public int Count
+      {
+         get { throw new Exception("The method or operation is not implemented."); }
+      }
+
+      public bool IsReadOnly
+      {
+         get { throw new Exception("The method or operation is not implemented."); }
+      }
+
+      public bool Remove(KeyValuePair<TKey, TValue> item)
+      {
+         throw new Exception("The method or operation is not implemented.");
+      }
+
+      #endregion
+
+      #region IEnumerable<KeyValuePair<TKey,TValue>> Members
+
+      public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
+      {
+         throw new Exception("The method or operation is not implemented.");
+      }
+
+      #endregion
+
+      #region IEnumerable Members
+
+      IEnumerator IEnumerable.GetEnumerator()
+      {
+         throw new Exception("The method or operation is not implemented.");
+      }
+
+      #endregion
+   }
+
    #region Helper class to deal with stream projections
 
    public class JoinProjectHelper
@@ -59,7 +167,7 @@ namespace BCG
       {
          Pair<string, bool> p = myAliasToOriginal[col];
          isLeft = p.second;
-         return ExceptionAssert.NotNull(p.first);
+         return p.first;
       }
    }
 
@@ -70,11 +178,10 @@ namespace BCG
    public class SuperJoinRow : IBlueRow
    {
       private JoinProjectHelper myHelper;
-      IBlueRow myLeftRow;
-      IBlueRow? myRightRow;
+      private IBlueRow myLeftRow, myRightRow;
       int myID;
 
-      public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
+      public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
       {
          myID = id;
          myLeftRow = leftRow;
@@ -82,22 +189,21 @@ namespace BCG
          myHelper = joinProjectHelper;
       }
 
-      public int leftID => myLeftRow.Id();
+      public int leftID { get { return (int)myLeftRow["id"]; } }
 
-      public int rightID => myRightRow.Id();
+      public int rightID { get { return (int)myRightRow["id"]; } }
 
       public IBlueRow leftRow { get { return myLeftRow; } }
 
-      public IBlueRow? rightRow => myRightRow;
+      public IBlueRow rightRow { get { return myRightRow; } }
 
       #region IBlueRow Members
 
       public IBlueTableSchema schema { get { return myHelper.schema; } }
 
-      public object? this[int index] =>
-         this[schema[index].colName];
+      public object this[int index] { get { return this[schema[index].colName]; } }
 
-      public object? this[string colName]
+      public object this[string colName]
       {
          get
          {
@@ -123,7 +229,7 @@ namespace BCG
 
       public bool Contains(object key)
       {
-         return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
+         return myLeftRow.Contains(key) || myRightRow.Contains(key);
       }
 
       public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
@@ -136,11 +242,11 @@ namespace BCG
 
       public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); }
 
-      public ICollection Values
+      public System.Collections.ICollection Values
       {
          get
          {
-            var rv = new List<object?>(Keys.Count);
+            List<object> rv = new List<object>();
             foreach (string col in Keys)
                rv.Add(this[col]);
             return rv;
@@ -185,10 +291,9 @@ namespace BCG
                                           myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
       private bool myManageUnmappedLeftRows = false;
       protected string myLeftProj = String.Empty, myRightProj = String.Empty;
-      protected IBlueTableSchema? myLeftSchema;
-      protected IBlueTableSchema? myRightSchema;
-      protected JoinProjectHelper? myJoinProjectHelper;
-      protected BlueTable? myState;
+      protected IBlueTableSchema myLeftSchema, myRightSchema;
+      protected JoinProjectHelper myJoinProjectHelper;
+      protected BlueTable myState;
       protected int myID;
       protected ViewStream myLeftStream, myRightStream;
       protected string myLeftKeyCol, myRightKeyCol;
@@ -241,19 +346,23 @@ namespace BCG
 
       private void myHandleLeftStreamChanges(object newValue)
       {
-         myHandleStreamChanges((IDataPacket)newValue, true);
+         myHandleStreamChanges(newValue as DataPacket, true);
       }
 
       private void myHandleRightStreamChanges(object newValue)
       {
-         myHandleStreamChanges((IDataPacket)newValue, false);
+         myHandleStreamChanges(newValue as DataPacket, false);
       }
 
-      private void myHandleStreamChanges(IDataPacket dp, bool isLeft)
+      private void myHandleStreamChanges(DataPacket dp, bool isLeft)
       {
-         myInitSchema(dp.Schema, isLeft);
-         var dels = new BlueTable(myState.schema);
-         var adds = new BlueTable(myState.schema);
+         myInitSchema(dp.schema, isLeft);
+         BlueTable dels = null, adds = null;
+         if (myState != null)
+         {
+            dels = new BlueTable(myState.schema);
+            adds = new BlueTable(myState.schema);
+         }
          myDeleteRows(dp.deletedRows, isLeft, myState, dels);
          myAddRows(dp.addedRows, isLeft, myState, adds);
          if (myState != null)
@@ -264,7 +373,6 @@ namespace BCG
          }
       }
 
-      [MemberNotNull(nameof(myState))]
       private void myInitSchema(IBlueTableSchema schema, bool isLeft)
       {
          if (myState == null)
@@ -294,7 +402,6 @@ namespace BCG
 
       private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
       {
-         ExceptionAssert.NotNull(myJoinProjectHelper);
          object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
             otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
          Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
@@ -303,8 +410,8 @@ namespace BCG
                                                          otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
          foreach (IBlueRow r in addedRows)
          {
-            object currentKey = ExceptionAssert.NotNull(r[thisKeyCol]);
-            int currentID = r.Id();
+            object currentKey = r[thisKeyCol];
+            int currentID = (int)r["id"];
             ///add it to 1.
             if (!thisKeyToRows.ContainsKey(currentKey))
                thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
@@ -315,14 +422,14 @@ namespace BCG
                ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
                foreach (IBlueRow mr in matchingRows)
                {
-                  int mrID = mr.Id();
+                  int mrID = (int)mr["id"];
                   SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
                   state.addRow(rr);
                   addedRowsOut.addRow(rr);
                   ///add to 3.
                   if (!thisIDToResultRows.ContainsKey(currentID))
                      thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
-                  thisIDToResultRows[currentID].Add(rr.Id(), rr);
+                  thisIDToResultRows[currentID].Add((int)rr["id"], rr);
                   ///add to 4.
                   if (!otherIDToResultRows.ContainsKey(mrID))
                   {
@@ -330,7 +437,7 @@ namespace BCG
                      if (myManageUnmappedLeftRows && !isLeft)
                         myTakeCareOfUnmatachedLeftRows(mr, false);
                   }
-                  otherIDToResultRows[mrID].Add(rr.Id(), rr);
+                  otherIDToResultRows[mrID].Add((int)rr["id"], rr);
                }
             }
             else
@@ -351,22 +458,22 @@ namespace BCG
                                                          otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
          foreach (IBlueRow r in deletedRows)
          {
-            int id = r.Id();
-            object key = ExceptionAssert.NotNull(r[thisKeyCol]);
+            int id = (int)r["id"];
+            object key = r[thisKeyCol];
             ///delete it from 1.
             thisKeyToRows[key].Remove(id);
             if (thisKeyToRows[key].Count == 0)
                thisKeyToRows.Remove(key);
             ///form return rows and delete from 3.
-            if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRowMap))
+            if (thisIDToResultRows.ContainsKey(id))
             {
-               var removedResultRows = (ICollection)removedResultRowMap.Values;
+               ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
                thisIDToResultRows.Remove(id);
                foreach (SuperJoinRow rr in removedResultRows)
                {
                   ///delete from 4.
                   int otherId = isLeft ? rr.rightID : rr.leftID;
-                  otherIDToResultRows[otherId].Remove(rr.Id());
+                  otherIDToResultRows[otherId].Remove((int)rr["id"]);
                   if (otherIDToResultRows[otherId].Count == 0)
                      otherIDToResultRows.Remove(otherId);
                   delRowsOut.addRow(rr);
@@ -386,7 +493,7 @@ namespace BCG
 
       private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
       {
-         int lid = leftRow.Id();
+         int lid = (int)leftRow["id"];
          if (add)
             myLeftUnmatchedRowsAdds.Add(lid, leftRow);
          else
@@ -394,7 +501,7 @@ namespace BCG
             if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
                myLeftUnmatchedRowsAdds.Remove(lid);
             else
-               myLeftUnmatchedRowsDels.Add(lid, leftRow);
+               myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
          }
       }
    }
@@ -415,49 +522,44 @@ namespace BCG
 
       protected override void myFireEvent(object o)
       {
-         myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, (IDataPacket)o);
+         myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket);
       }
 
-      void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels, Dictionary<int, IBlueRow> unmatchedLeftRowAdds, IDataPacket resultPacket) {
-         ExceptionAssert.NotNull(myJoinProjectHelper);
-         var dels = resultPacket.deletedRows;
-         var adds = resultPacket.addedRows;
+      private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
+                                       Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
+      {
+         BlueTable dels = resultPacket.deletedRows as BlueTable,
+                     adds = resultPacket.addedRows as BlueTable;
 
-         if (unmatchedLeftRowsDels.Count > 0) {
-            ExceptionAssert.NotNull(myState);
-            var mutableDels = (BlueTable)dels;
-            foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
-               ///an lr will be here if its actually deleted or
-               ///a right row is added, so now it has a partner
-               ///and is no longer unmatched.
-               ///in both cases the half row that this lr
-               ///created needs to be removed from this
-               ///class's state and added to dels
-               ///it means a right is added
-               int lid = lr.Id();
-               SuperJoinRow rr = myLeftIDToHalfRows[lid];
-               myLeftIDToHalfRows.Remove(lid);
-               myState.deleteRow(rr);
-               mutableDels.addRow(rr);
-            }
+         foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
+         {
+            ///an lr will be here if its actually deleted or
+            ///a right row is added, so now it has a partner
+            ///and is no longer unmatched.
+            ///in both cases the half row that this lr
+            ///created needs to be removed from this
+            ///class's state and added to dels
+            ///it means a right is added
+            int lid = (int)lr["id"];
+            SuperJoinRow rr = myLeftIDToHalfRows[lid];
+            myLeftIDToHalfRows.Remove(lid);
+            myState.deleteRow(rr);
+            dels.addRow(rr);
          }
 
-         if (unmatchedLeftRowAdds.Count > 0) {
-            ExceptionAssert.NotNull(myState);
-            var mutableAdds = (BlueTable)adds;
-            foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
-               ///an lr will be here if its added with no 
-               ///matching right, or a right matching row
-               ///is gone so it is unmatched now.
-               ///in both cases a half row needs to be
-               ///added to this class's state and added 
-               ///to adds
-               int lid = lr.Id();
-               SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
-               myLeftIDToHalfRows.Add(lid, rr);
-               myState.addRow(rr);
-               mutableAdds.addRow(rr);
-            }
+         foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
+         {
+            ///an lr will be here if its added with no 
+            ///matching right, or a right matching row
+            ///is gone so it is unmatched now.
+            ///in both cases a half row needs to be
+            ///added to this class's state and added 
+            ///to adds
+            int lid = (int)lr["id"];
+            SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
+            myLeftIDToHalfRows.Add(lid, rr);
+            myState.addRow(rr);
+            adds.addRow(rr);
          }
          base.myFireEvent(new DataPacket(myState, adds, dels));
       }

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -1,4 +1,6 @@
 namespace BCG {
+    using System.Collections.Generic;
+
     public class KeyValueToRelationalStream : ViewStream {
         private readonly ViewStream _kv_str;
         private readonly string _doc_id_col, _key_col, _val_col;
@@ -18,7 +20,7 @@ namespace BCG {
         }
 
         private void _on_str_update(object dp_obj) {
-            var dp = (IDataPacket)dp_obj;
+            DataPacket dp = dp_obj as DataPacket;
             var rows = dp.newRows;
             var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
             var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
@@ -26,4 +28,4 @@ namespace BCG {
         }
 
     }
-}
+}

src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs

@@ -1,9 +1,6 @@
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-
 using System;
-using System.Collections.Generic;
+using System.Collections;
+using BCG.Utilities;
 
 namespace BCG
 {
@@ -14,11 +11,9 @@ namespace BCG
    [DataStreamUserRespresentation (false)]
    public class LeftInternalJoinStream : InternalJoinStream
    {
-      protected IDataPacket? myLeftDataPacket;
-      protected IDataPacket? myRightDataPacket;
+      protected DataPacket myLeftDataPacket, myRightDataPacket;
       //Container to contain all the left rows that are orphaned during a deletion operation
-      Dictionary<int, RowNode>? myLeftOrphanedRows;
-
+      private IDictionary myLeftOrphanedRows;
       public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) :
          base (vsLeft, jcLeft, vsRight, jcRight)   {}
 
@@ -67,9 +62,7 @@ namespace BCG
       private void myTestAndFire (bool isLeft)
       {
          JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
-         var dp = isLeft
-            ? ExceptionAssert.NotNull(myLeftDataPacket)
-            : ExceptionAssert.NotNull(myRightDataPacket);
+         DataPacket dp = isLeft ? myLeftDataPacket : myRightDataPacket;
          //to test whether both have received call backs
          if (myLeftOperand.hasReceivedData)
             jo.changeHandler (dp);
@@ -87,13 +80,11 @@ namespace BCG
       /// <param name="isLeft"></param>
       public override void addRows (IBlueRowCollection rows, bool isLeft)
       {
-         ExceptionAssert.NotNull(myLastFiredPacket);
          foreach (IBlueRow row in rows)
             myAddRow (row, isLeft);
          if (!isLeft)
-            foreach (RowNode node in ExceptionAssert.NotNull(myLeftOrphanedRows).Values) {
-               ((JoinTable)myLastFiredPacket.newRows).addRow(node.RequireRow(), null, myLastFiredPacket);
-            }
+            foreach (RowNode node in myLeftOrphanedRows.Values)
+               ((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket);
       }
 
       /// <summary>
@@ -103,19 +94,17 @@ namespace BCG
       /// <param name="isLeft"></param>
       public override void deleteRows (IBlueRowCollection rows, bool isLeft)
       {
-         myLeftOrphanedRows = new();
+         myLeftOrphanedRows = new Hashtable ();
          foreach (IBlueRow row in rows)
             myDeleteRow (row, isLeft);
       }
 
       private void myAddRow (IBlueRow row, bool isLeft)
       {
-         ExceptionAssert.NotNull(myLastFiredPacket);
          if (isLeft)
          {
             myLeftOperand.addRow (row);
-            var joinKey = ExceptionAssert.NotNull(row[myLeftJoinColOrdinal]);
-            RowList matchedRows = myRightOperand.getMatchingRows(joinKey);
+            RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]);
             if (matchedRows.Count == 0)
                ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
             else 
@@ -124,25 +113,22 @@ namespace BCG
          }
          else
          {
-            var joinKey = ExceptionAssert.NotNull(row[myRightJoinColOrdinal]);
-            RowList matchedRows = myLeftOperand.getMatchingRows(joinKey);
-            int rightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count;
+            RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
+            int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
             myRightOperand.addRow (row);
-            int newRightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count;
+            int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
             if (matchedRows.Count != 0){
                foreach (RowNode matchedNode in matchedRows)
                {
-                  ExceptionAssert.NotNull(matchedNode.row);
                   //if the left rows were orphaned in this transaction (ie in this callback), they should
                   //just be removed from the orphanedlist and we dont need to worry about the consistency of
                   //the jointable as a new row will replace it
                   if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0)
                   {
-                     ExceptionAssert.NotNull(myLeftOrphanedRows);
-                     int matchedId = matchedNode.row.Id();
-                     if (!myLeftOrphanedRows.Remove(matchedId)) {
-                        ((JoinTable)myLastFiredPacket.newRows).deleteRow(true, matchedNode.row, myLastFiredPacket);
-                     }
+                     if (myLeftOrphanedRows[matchedNode.row["id"]] == null)
+                        ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket);
+                     else
+                        myLeftOrphanedRows.Remove (matchedNode.row["id"]);
                   }
                   //they were orphaned by a previous transaction and hence the orphans need to be
                   //removed from the Jointable and the complete row added
@@ -155,52 +141,41 @@ namespace BCG
 
       private void myDeleteRow (IBlueRow row, bool isLeft)
       {
-         var last = requireLastFiredPacket();
          if (isLeft)
          {
             myLeftOperand.deleteRow (row);
-            ((JoinTable)last.newRows).deleteRow (true, row, myLastFiredPacket);
+            ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, row, myLastFiredPacket);
          }
          else
          {
-            var rightJoinKey = row[myRightJoinColOrdinal] ?? throw new InvalidOperationException("RightJoinKey is null");
-            RowList matchedRows = myLeftOperand.getMatchingRows(rightJoinKey);
-            int rightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count;
+            RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
+            int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
             myRightOperand.deleteRow (row);
-            int newRightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count;
+            int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
             if (matchedRows.Count != 0)
             {
-               if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) {
-                  ExceptionAssert.MemberNotNull(myLeftOrphanedRows);
+               if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0)
                //ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to
                //be handled specially, ie joined with nulls, if there are no right rows with this
                //joincolumn value being inserted. 
-                  ExceptionAssert.NotNull(myLeftOrphanedRows);
                   foreach (RowNode matchedNode in matchedRows)
                   {
                      // BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null);
-                     myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode;
+                     myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode;
                   }
-               }
-               ((JoinTable)last.newRows).deleteRow (false, row, myLastFiredPacket);
+               ((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket);
             }
          }           
       }
       
       public override void onTableClear (bool isLeft)
       {
-         if (myLastFiredPacket is null) {
-            return;
-         }
-         requireSchema();
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         if (!isLeft) {
-            ExceptionAssert.MemberNotNull(myLeftDataPacket);
+         if (!isLeft)
             foreach (IBlueRow row in myLeftDataPacket.newRows)
                ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
-         }
       }
       
    }

src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs

@@ -30,7 +30,7 @@ namespace BCG
       /// <summary>
       /// Node containing the referencedRow
       /// </summary>
-      IBlueRow? referencedRow { get; }
+      IBlueRow referencedRow {get;}
 
       /// <summary>
       /// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row