CLEANUP: BYBIT Symbols.conf.

Originally, this spark.conf was hardcoded into compose.yaml, and copied
into the sparkd container on each deploy. It hasn't been used since
config profiles were introduced.

.config/dotnet-tools.json

@@ -25,12 +25,6 @@
       "commands": [
         "fake"
       ]
-    },
-    "dotnet-t4": {
-      "version": "3.0.0",
-      "commands": [
-        "t4"
-      ]
     }
   }
 }

.devcontainer/Dockerfile

@@ -55,16 +55,14 @@ RUN \
    clangd-$clang_major
 
 # CMake
-RUN set -eux \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends ca-certificates curl gpg \
-  && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
-    | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
-  && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-    > /etc/apt/sources.list.d/kitware.list \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends cmake \
-  && rm -rf /var/lib/apt/lists/*
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
 
 # Dotnet
 RUN apt-get update && apt-get install -y dotnet8
@@ -82,14 +80,9 @@ RUN mkdir -p /etc/apt/keyrings \
 # Emumba (deps)
 RUN apt-get update && apt-get -y install rapidjson-dev
 # Emumba
-ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
-RUN --mount=type=secret,id=bitbucket_access_token \
-  set -eux \
-    && apt-get update \
-    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
-    && curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
-    && apt-get install -y /tmp/emumba-deribit-connector.deb \
-    && rm -f /tmp/emumba-deribit-connector.deb
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
 
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -97,18 +90,6 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
-# Rakurai
-ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
-RUN --mount=type=secret,id=bitbucket_access_token \
-  set -eux \
-    && apt-get update \
-    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
-    && curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
-    # Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
-    # spdlog headers.
-    && apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
-    && rm -f /tmp/rakuirai.deb
-
 # Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \

.devcontainer/devcontainer.json

@@ -1,11 +1,5 @@
 {
-  "build": {
-    "dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
-    "options": [
-      "--secret",
-      "id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
-    ]
-  },
+  "dockerFile": "Dockerfile",
   "hostRequirements": {
     "memory": "8gb",
     "storage": "10gb"
@@ -28,7 +22,6 @@
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
-        "ms-python.python",
         "ms-vscode.cmake-tools",
         "ms-vscode.cpptools",
         "xaver.clang-format"

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.editorconfig

@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles
 
-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower
 
-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion

.env

@@ -29,36 +29,20 @@
 #     tools/deploy.sh --help
 #
 
-### Host/Runtime ###
-TZ=Etc/UTC
-
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
 #   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
 # The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
-# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
-# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
-# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
-SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
 
-### PubSubServer ###
-PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
-PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-
-### MdPublisher ###
-MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
-MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -11,7 +11,6 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
-!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -55,8 +54,6 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
-/src/**/*appsettings.Local.json
-/src/**/*/Configurations/Local/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/

.vscode/launch.json

@@ -14,49 +14,6 @@
       "type": "coreclr",
       "request": "attach"
     },
-    {
-      "name": "Api Server",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
-      "windows": {
-        "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Api Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -69,28 +26,11 @@
       "windows": {
         "program": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
       "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DataStreamPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -142,30 +82,17 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "DataStreamQueryAnalyzer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -183,65 +110,11 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
       "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PubSubServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sdk Demo",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "preLaunchTask": "Build Spark Sdk (Debug)",
-      "windows": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "linux": {
-        "program": "Bts.Spark.Sdk.Demo",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sdk Demo (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "linux": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -254,9 +127,6 @@
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -269,9 +139,6 @@
       "windows": {
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -285,7 +152,7 @@
       "windows": {
         "program": "Spark.exe",
         "args": [
-          "--Spark:ConfigServer",
+          "--sparkd",
           "${input:sparkConfigServer}"
         ],
         "cwd": "build/Debug/bin/net8.0-windows",
@@ -333,7 +200,6 @@
       "program": "",
       "linux": {
         "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-        "preLaunchTask": "Build Sparkd (Debug | Clang)",
       },
       "args": [
         "run"
@@ -383,38 +249,8 @@
         }
       ]
     },
-    {
-      "name": "Sparkd Attach",
-      "type": "cppdbg",
-      "request": "attach",
-      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
   ],
   "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "dataStreamPublisherConfig",
       "type": "pickString",
@@ -428,46 +264,14 @@
       "default": "MarketDataPublisher",
     },
     {
-      "id": "dataStreamQueryAnalyzerEnvironment",
+      "id": "pubSubServerConfig",
       "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
+      "description": "PubSub Configuration",
       "options": [
-        "Development",
-        "Local",
+        "PubSubServer",
+        "SimpleEchoClient",
       ],
-      "default": "Development",
-    },
-    {
-      "id": "dataStreamPublisherEnvironment",
-      "type": "pickString",
-      "description": "DataStreamPublisher Configuration",
-      "options": [
-        "Local",
-        "ReplayMarketData",
-      ],
-      "default": "Local",
-    },
-    {
-      "id": "pubSubServerEnvironment",
-      "type": "pickString",
-      "description": "PubSubServer Configuration",
-      "options": [
-        "Development",
-        "EchoServer",
-        "Local",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "sdkDemoEnvironment",
-      "type": "pickString",
-      "description": "Sdk Demo Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
+      "default": "PubSubServer",
     },
     {
       "id": "sparkConfigServer",
@@ -475,16 +279,6 @@
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
-    {
-      "id": "sparkEnvironment",
-      "type": "pickString",
-      "description": "Spark Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
     {
       "id": "sparkTest",
       "type": "promptString",

.vscode/python.env

@@ -1 +0,0 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py

.vscode/tasks.json

@@ -172,66 +172,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Build Spark Sdk",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark Sdk (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark Sdk (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Build Sparkd",
       "group": "build",
@@ -411,129 +351,6 @@
         "id": "wrench"
       },
     },
-    {
-      "label": "Run Api Server",
-      "dependsOn": "Build Api Server",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug)",
-      "dependsOn": "Build Api Server (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release)",
-      "dependsOn": "Build Api Server (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
     {
       "label": "Run DataStreamQueryAnalyzer",
       "dependsOn": "Build DataStream",
@@ -541,11 +358,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -560,11 +378,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -579,11 +398,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -602,11 +422,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -624,11 +445,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -646,11 +468,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -668,11 +491,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -690,11 +514,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -711,11 +536,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -733,11 +559,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -754,11 +581,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -773,7 +601,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -789,7 +617,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -805,7 +633,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -820,7 +648,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -845,7 +673,7 @@
         "${input:sparkConfigQuery}",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -926,7 +754,7 @@
           "${input:sparkTest}"
         ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
         }
       },
     },
@@ -1024,17 +852,6 @@
     },
   ],
   "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "buildConfiguration",
       "type": "pickString",
@@ -1045,16 +862,6 @@
       ],
       "default": "Debug",
     },
-    {
-      "id": "dataStreamQueryAnalyzerEnvironment",
-      "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
     {
       "id": "sparkdCmakePreset",
       "type": "pickString",
@@ -1078,30 +885,26 @@
       "default": "sparkd-debug-clang",
     },
     {
-      "id": "dataStreamPublisherEnvironment",
+      "id": "dataStreamPublisherConfig",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "Development",
-        "EchoServer",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "PubSubServer",
+        "SimpleEchoClient",
       ],
-      "default": "Development",
+      "default": "PubSubServer",
     },
     {
       "id": "sparkConfigQuery",

Bts.Spark.Sdk.sln

@@ -1,102 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|Any CPU = Release|Any CPU
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.Build.0 = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.Build.0 = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.Build.0 = Release|x64
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.Build.0 = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-	EndGlobalSection
-EndGlobal

CMakeLists.txt

@@ -21,7 +21,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -40,8 +40,9 @@ include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include
 link_directories(${EXTERNALS_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-if(DEFINED APP_DATA_DIR)
-  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
@@ -112,15 +113,15 @@ else()
   endif()
 
   # subdirs
-  add_subdirectory(src/lib_spark)
   if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
     add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance)
+    add_subdirectory(src/lib_binance_execution)
     add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/lib_python_md)
     add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)
   endif()
+  add_subdirectory(src/lib_spark)
   if (BUILD_ADMIN_WEB_APP)
     add_subdirectory(src/Spark.Admin.WebApp)
   endif()

CMakePresets.json

@@ -8,41 +8,36 @@
 
   "configurePresets": [
     {
-      "name": "spark-base",
-      "hidden": true,
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
-      }
-    },
-
-    {
-      "name": "spark-windows",
-      "inherits": "spark-base",
-      "displayName": "Spark",
+      "name": "spark",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
+      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
-        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
       }
     },
 
@@ -58,6 +53,8 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "OFF"
@@ -80,7 +77,8 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
       }
     },
 
@@ -159,17 +157,22 @@
     },
 
     {
-      "name": "spark-linux",
-      "inherits": "spark-base",
-      "displayName": "Spark",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
+      "name": "spark-api-server-windows",
+      "inherits": "spark",
+      "displayName": "Spark Api Server (Windows)",
+      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
+      }
+    },
+
+    {
+      "name": "spark-api-server-linux",
+      "inherits": "sparkd",
+      "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "platform": "linux",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -178,26 +181,21 @@
           "type": "BOOL",
           "value": "OFF"
         },
-        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_EXPORT_COMPILE_COMMANDS": {
           "type": "BOOL",
           "value": "ON"
         },
-        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
-        "DATABENTO_USE_EXTERNAL_JSON": {
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "EXTERNALS_ROOT": "${sourceDir}/externals/linux"
+        }
       }
     },
 
@@ -217,32 +215,92 @@
       "cacheVariables": {
         "CCAPI_LOG_LEVEL": "Trace"
       }
-    }
-  ],
+    },
 
-  "buildPresets": [
     {
-      "name": "spark-windows",
+      "name": "data-stream-suite-windows",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
-      "hidden": true,
-      "configurePreset": "spark-windows"
+      "inherits": "spark",
+      "displayName": "Data Stream Suite (Windows)",
+      "description": "Data Stream utils ported over from Edge",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
     },
 
     {
-      "name": "spark-windows-debug",
+      "name": "data-stream-suite-linux",
+      "displayName": "Data Stream Suite (Linux)",
+      "description": "Data Stream utils ported over from Edge",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "inherits": "sparkd",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
+    }
+  ],
+
+  "buildPresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true
+    },
+
+    {
+      "name": "spark-debug",
       "displayName": "Spark (Debug)",
-      "inherits": "spark-windows",
+      "inherits": "spark",
+      "configurePreset": "spark",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-windows-release",
+      "name": "spark-release",
       "displayName": "Spark (Release)",
-      "inherits": "spark-windows",
+      "inherits": "spark",
+      "configurePreset": "spark",
       "configuration": "Release"
     },
 
@@ -285,27 +343,52 @@
     },
 
     {
-      "name": "spark-linux",
+      "name": "spark-api-server-windows",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true,
+      "configurePreset": "spark-api-server-windows"
+    },
+
+    {
+      "name": "spark-api-server-windows-debug",
+      "displayName": "Spark Api Server (Windows | Debug)",
+      "inherits": "spark-api-server-windows",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "spark-api-server-windows-release",
+      "displayName": "Spark Api Server (Windows | Release)",
+      "inherits": "spark-api-server-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "spark-api-server-linux",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
       "hidden": true,
-      "configurePreset": "spark-linux"
+      "configurePreset": "spark-api-server-linux"
     },
 
     {
-      "name": "spark-linux-debug",
-      "inherits": "spark-linux",
-      "displayName": "Spark (Linux | Debug)",
+      "name": "spark-api-server-linux-debug",
+      "inherits": "spark-api-server-linux",
+      "displayName": "Spark Api Server (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-linux-release",
-      "inherits": "spark-linux",
-      "displayName": "Spark (Linux | Release)",
+      "name": "spark-api-server-linux-release",
+      "inherits": "spark-api-server-linux",
+      "displayName": "Spark Api Server (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -322,6 +405,34 @@
       "configurePreset": "sparkd-ccapi-diag",
       "displayName": "Debug (ccapi diag)",
       "inherits": "sparkd"
+    },
+
+    {
+      "name": "data-stream-suite-windows-debug",
+      "displayName": "Data Stream Suite (Windows | Debug)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-windows-release",
+      "displayName": "Data Stream Suite (Windows | Release)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "data-stream-suite-linux-debug",
+      "displayName": "Data Stream Suite (Linux | Debug)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-linux-release",
+      "displayName": "Data Stream Suite (Linux | Release)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Release"
     }
   ]
 }

Spark.Api.Server.sln

@@ -1,11 +1,10 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
@@ -43,538 +42,168 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
-		MinSizeRel|Any CPU = MinSizeRel|Any CPU
-		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
-		Release|Any CPU = Release|Any CPU
-		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
-		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
-		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.Build.0 = Debug|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.ActiveCfg = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.Build.0 = Release|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.Build.0 = Debug|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.ActiveCfg = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.Build.0 = Release|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.Build.0 = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.Build.0 = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.Build.0 = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.Build.0 = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.Build.0 = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.Build.0 = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.Build.0 = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.Build.0 = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.Build.0 = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.Build.0 = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.Build.0 = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.Build.0 = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.Build.0 = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.Build.0 = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.Build.0 = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.Build.0 = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.Build.0 = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.Build.0 = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.Build.0 = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.Build.0 = Release|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
@@ -586,17 +215,8 @@ Global
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
 	EndGlobalSection
 EndGlobal

cmake/InitPythonModule.cmake

@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()

cmake/InitPythonPlugin.cmake

@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()

cmake/Protobuf.cmake

@@ -0,0 +1,164 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    else()
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    endif()
+
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+
+endfunction()
+

cmake/SymLinkPythonModule.cmake

@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()

compose.yaml

@@ -1,44 +1,27 @@
 services:
   sparkd:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
         - IMAGE=${SPARKD__IMAGE}
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-shell:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-    build:
-      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: "do sleep 1; done"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -53,18 +36,30 @@ services:
 
   sparkd-symbols:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
     build:
-      dockerfile: src/app_sparkd/Dockerfile
+      dockerfile: src/server/sparkd/Dockerfile
       args:
         - IMAGE=${SPARKD__IMAGE}
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -72,7 +67,6 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       restart_policy:
         condition: any
@@ -80,75 +74,17 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-  sparkd-multicast:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
-  
-  sparkd-multicast-cert:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
+
   sparkd-tail:
-    image: bluetradingsystems/linux_dotnet8
+    image: bluetradingsystems/linux_dotnet7
     volumes:
       - type: volume
         read_only: true
         source: sparkd_data
-        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
+        target: /home/bts/sparkdata/data/logging.sparkd
         volume:
           subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
 
   sparkd-ping-test:
     depends_on:
@@ -158,97 +94,11 @@ services:
     build:
       context: .
       dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet8
+        FROM bluetradingsystems/linux_dotnet7
         USER "root"
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
 
-  pubsub:
-    volumes:
-      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
-        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
-    ports:
-      - 22019:22019
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
-        if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
-          mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/pubsub/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/pubsub/conf/.env.local
-          set +a
-        fi
-        ./PubSubServer"
-    env_file:
-      - .env
-      - ${PUBSUB__CONFIG_DIR}/.env
-
-  md-publisher:
-    volumes:
-      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
-      - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
-    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
-        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
-        if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
-          mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/md_publisher/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/md_publisher/conf/.env.local
-          set +a
-        fi
-      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
-    env_file:
-      - .env
-      - ${MD_PUBLISHER__CONFIG_DIR}/.env
-
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
-
 volumes:
   sparkd_data:
     name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
-  pubsub_data:
-    name: "pubsub_22019"
-  md_publisher_data:
-    name: "md_publisher_22019"
-  historic_md:
-    name: "historic_md"
-  md_recording:
-    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
-    external: true

doc/sparkd-security-id-lifecycle.md

@@ -1,942 +0,0 @@
-# Security ID handling in `sparkd`
-
-This document traces how security identifiers are created, translated, persisted,
-indexed, and published by the `sparkd` market-data paths. It focuses on the
-legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
-
-## Executive summary
-
-`sparkd` uses two identifiers that must not be confused:
-
-| Identifier | Meaning | Typical field | Lifetime |
-|---|---|---|---|
-| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
-| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
-
-`invalid_security_key` is `0`; a security with key `0` is not yet publicly
-discoverable through the shared-memory consumer path.
-
-The two principal paths differ:
-
-- **MDP3/OnixS:** the definition callback sets only the CME ID in
-  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
-  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
-  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
-  calls `MDProducerHandle::set_key()`.
-- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
-  currently uses the ID already present on the incoming domain security
-  (`use_given_sec_key == true`). Live adapters normally obtain that ID
-  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
-  it to both `FixedSecurity::key` and `numeric_id`, so
-  `MDProducer::add_security()` publishes it synchronously. Only the listener's
-  own fallback `IDGenerator` call is bypassed by the current constant.
-
-## End-to-end diagrams
-
-### CME MDP3 / OnixS
-
-```mermaid
-sequenceDiagram
-    participant FE as OnixS feed-engine thread
-    participant L as MDP3Listener
-    participant P as MDProducer / shared memory
-    participant Q as SecurityIDRecorder queues
-    participant D as sparkd dispatcher thread
-    participant DB as key_ids.sqlite
-
-    FE->>L: instrument definition (CME SecurityID)
-    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
-    L->>P: add_security(FixedSecurity)
-    P->>P: allocate hot + cold entries with key 0
-    P->>Q: ensure_key(handle)
-    L->>L: index[CME SecurityID] = handle
-    Note over FE,L: Books can route by CME ID while the Spark key is pending
-    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
-    Q->>DB: find (namespace, descriptor), or insert a new row
-    DB-->>Q: SQLite rowid = Spark security key
-    Q->>P: handle.set_key(rowid)
-    P->>P: hot.key first, then cold security.key, then increment security_count
-    Note over P: The security is now public to shared-memory consumers
-```
-
-### Generalized/newer pipeline (current behavior)
-
-```mermaid
-sequenceDiagram
-    participant A as Venue adapter / plugin
-    participant DB as IDGenerator / key_ids.sqlite
-    participant G as GeneralMarketDataListener
-    participant I as GeneralSecurityIndex
-    participant P as MDProducer / shared memory
-
-    A->>DB: get_or_add(venue, type, adapter key)
-    DB-->>A: SQLite rowid as domain SecurityID
-    A->>G: on_securities(domain Security list with ID)
-    G->>G: _find_sec() - use incoming security_common.id
-    G->>G: _copy_security_common() - key = id, numeric_id = id
-    G->>P: add_security(FixedSecurity with non-zero key)
-    P->>P: set_key() during add_security()
-    G->>I: index[id] = handle
-    Note over G,P: No asynchronous ID assignment in the current configuration
-```
-
-## Data-flow charts
-
-The sequence diagrams above emphasize timing and thread transitions. These
-flowcharts emphasize which identifier is carried through each representation
-and where data is retained.
-
-### MDP3 / legacy data flow
-
-```mermaid
-flowchart TD
-    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
-
-    ONIXS --> DEF["MDP3Listener definition callback"]
-    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
-
-    FIXED --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
-    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
-
-    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
-
-    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
-    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
-    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
-    UNRESOLVED --> CMEINDEX
-    CMEINDEX --> LEGS
-    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
-
-    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
-    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
-    INSERT --> ROWID
-
-    ROWID --> SETKEY["MDProducerHandle::set_key"]
-    SETKEY --> HOT["Write hot-entry key"]
-    HOT --> COLD["Write cold security key"]
-    COLD --> COUNT["Increment security_count"]
-    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
-
-    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
-    UPDATE --> CMEINDEX
-    CMEINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> LIVE["Update live shared-memory data"]
-    PUBLIC --> LIVE
-```
-
-The CME index is usable before the Spark key is assigned, so feed updates can
-reach the internal handle while the security is still hidden from consumers.
-The persistent rowid crosses the publication boundary only at `set_key()`.
-
-### Generalized/newer data flow
-
-```mermaid
-flowchart TD
-    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
-
-    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
-    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
-    IDINPUT --> MUTEX["IDGenerator global mutex"]
-    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid"]
-    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
-    INSERT --> ROWID
-    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
-    IDTYPE -- Recorded or pre-keyed --> DOMAINID
-
-    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
-    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
-    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
-    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
-
-    CONVERT --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
-    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
-    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
-
-    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
-    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
-    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
-    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
-    PYHOT --> EVENTID
-    PYHOT -. uses same mutex and SQLite path .-> MUTEX
-    EVENTID --> CCMAP
-    CCMAP --> GENERALINDEX
-    GENERALINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
-    SHM --> SHMUPDATE
-
-```
-
-In the normal newer live path, persistence occurs before the definition enters
-the generalized listener. The DB call therefore adds synchronous adapter-side
-latency, but publication needs no later timer. Most adapters use an in-memory
-map for subsequent events; the dotted Python path is the documented exception.
-
-## The publication boundary
-
-`MDProducerHandle::set_key()` is the definitive publication point for the
-legacy shared-memory representation. Its write ordering is intentional:
-
-1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
-2. Write the key into the cold security entry.
-3. Increment the cold header's public `security_count`.
-
-The cold security key is written last because consumers treat a non-zero cold
-key as visibility. At that moment the corresponding hot entry must already be
-addressable. `set_key()` only permits the transition from `0` to one key; it is
-not a key-update mechanism.
-
-## Identifier representations
-
-The names are historically inconsistent. In particular, many protobuf fields
-named `security_id` contain the **Spark key**, not the exchange's ID.
-
-| Layer | Exchange/feed identifier | Spark identifier |
-|---|---|---|
-| MDP3 wire/OnixS | `securityId()` | not present |
-| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
-| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
-| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
-| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
-| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
-| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
-| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
-
-The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
-shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
-64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
-allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
-no explicit checked conversion.
-
-## Which parts use exchange IDs, descriptors, or Spark security IDs?
-
-The three identifier classes serve different jobs:
-
-- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
-- **Descriptor/natural key:** recognizes a definition logically at the
-  persistence, discovery, or human-facing boundary.
-- **Spark security ID/key:** joins the instrument to books, orders, fills,
-  positions, and services after publication.
-
-The core downstream system does not repeatedly identify a security from its
-description. The description is primarily used to obtain a Spark key; after
-that boundary, most joins use the numeric Spark key.
-
-### Component-level usage matrix
-
-| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
-|---|---|---|---|
-| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
-| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
-| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
-| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
-| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
-| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
-| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
-| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
-| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
-| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
-| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
-| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
-| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
-| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
-| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
-| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
-| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
-| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
-| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
-| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
-
-### Exchange/external IDs: where they stop
-
-In MDP3 the separation is explicit:
-
-```text
-CME SecurityID
-    -> FixedSecurity.numeric_id
-    -> CMESecurityIndex lookup
-    -> MDProducerHandle
-    -> Spark security_key for downstream publication
-```
-
-The external CME ID continues to route feed messages but does not become the
-order/position key. This permits the same downstream Spark key to survive a
-restart as long as the logical descriptor mapping remains in
-`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
-
-The generalized path collapses more of these roles:
-
-```text
-exchange/vendor identity
-    -> adapter calls IDGenerator
-    -> domain::SecurityID
-    -> FixedSecurity.key and FixedSecurity.numeric_id
-    -> GeneralSecurityIndex and downstream Spark key
-```
-
-`SecurityCommon::exchange_sec_id` may still contain the original textual
-external identifier, but `GeneralMarketDataListener` does not use it to route
-events and does not put it in `numeric_id`. The adapter must translate later
-events to `domain::SecurityID` before invoking the listener.
-
-### Descriptor/name users
-
-Descriptors are used in three main places:
-
-1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
-   descriptor, while `IDGenerator` uses its caller-provided `name`.
-2. **Definition lookup and discovery:** `SecurityCache` supports
-   venue/type/name lookup; symbol filters and administrative tools use security
-   metadata to select or display instruments.
-3. **Diagnostics:** logs include symbols and descriptions so humans can
-   understand a numeric key.
-
-Descriptions generally do not drive the high-rate book, trade, order, fill, or
-position paths. Those paths use numeric IDs after definition resolution.
-
-It is important that “descriptor” does not mean one universal canonical field:
-
-- Legacy CME uses Globex `security_desc` from tag 55.
-- Many generalized adapters use `sec.desc()` or an exchange instrument name.
-- Databento uses its numeric instrument ID converted to a string.
-- Python uses `exchange_security_id`.
-
-Therefore, two adapters may describe the same economic contract but request
-different logical rows and receive different Spark IDs.
-
-### Spark security-ID users
-
-Once a security is published, its Spark key is the dominant system identifier:
-
-```mermaid
-flowchart LR
-    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
-    KEY --> MDC["MDConsumer and book subscriptions"]
-    KEY --> MBO["MBO snapshots and queries"]
-    KEY --> OMS["Orders, algorithms, and spread legs"]
-    KEY --> FILLS["Fills and execution records"]
-    KEY --> POS["Positions, PNL, and risk"]
-    KEY --> API["Services and client APIs"]
-    KEY --> REC["Relay and market-data recordings"]
-    KEY --> SECDB["Serialized security definitions"]
-```
-
-Fields named `security_id`, `security_key`, or top-level `Security.id()` in
-these downstream areas almost always mean the Spark key. They should not be
-compared directly with CME `SecurityID`, Databento `instrument_id`, or another
-venue/vendor identifier without going through the relevant adapter/index
-mapping.
-
-## Persistence and allocation
-
-### The authoritative key mapping: `key_ids.sqlite`
-
-Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
-file is:
-
-```text
-<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
-```
-
-The database contains one table and two indexes:
-
-```sql
-CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
-CREATE UNIQUE INDEX key_index ON store(ns, key);
-CREATE INDEX ns_index ON store(ns);
-```
-
-The SQLite `rowid` is the generated Spark key. `ns` is normally
-`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
-descriptor chosen by the caller. `value` is normally empty. Therefore ID
-stability depends on all three of these remaining stable:
-
-- the configured `data_path` and preservation of this database;
-- the venue/type namespace;
-- the descriptor supplied by the feed adapter.
-
-The two allocator implementations share the file and convention:
-
-- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
-  uses `securities::descriptor()`. For CME this is the Globex
-  `security_desc`, populated from the definition's symbol/tag 55.
-- `IDGenerator` (newer): derives the namespace from new domain enums and uses
-  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
-  identifier; Databento deliberately passes the decimal instrument ID, and the
-  Python bridge uses the exchange security ID.
-
-### Logical-row examples
-
-The following names are illustrative. Production uses the exact string emitted
-by the selected feed/adapter, including its case, spacing, and weekly-series
-notation.
-
-#### Legacy `SecurityIDRecorder`
-
-For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
-Globex `security_desc`. After converting the fixed security to
-`api::Security`, `securities::descriptor()` returns that same
-`glbx.security_desc()` string.
-
-| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
-|---|---|---|---|
-| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
-| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
-| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
-| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
-
-A weekly option does not get a special namespace. Standard and weekly options
-are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
-`security_desc` distinguishes the weekly series. For example, if the database
-assigned rowids `42001` through `42004` to the rows above, those integers—not
-the descriptor strings—would be the Spark security keys.
-
-Conceptually, legacy allocation is:
-
-```text
-future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
-option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
-weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
-spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
-```
-
-#### Newer `IDGenerator`
-
-The namespace result is similar, but the third argument is chosen by the
-adapter. Examples using a description-oriented adapter are:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
-// store: ns="GLBX.FUTURE", key="ESZ6"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6 C6500"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
-
-get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
-// store: ns="COINBASE.SPOT", key="BTC-USD"
-```
-
-Databento intentionally produces a different logical row even when the human
-description is `ESZ6`. If its instrument ID were `12345678`, it would call the
-equivalent of:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
-// store: ns="GLBX.FUTURE", key="12345678"
-```
-
-Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
-`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
-while a provider that sends `exchange_security_id="BTCUSD"` produces a
-different row and therefore a different Spark key.
-
-These examples show why the tuple is only logical when the adapter's naming
-contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
-describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
-
-Both enum-prefix removals currently produce the same namespace spelling (for
-example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
-rowid space. Live rows cannot share a rowid. The table does not declare
-`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
-SQLite to reuse a number later. Security-key rows should therefore be treated
-as append-only operational data.
-
-`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
-and feed latency over maximum durability against a host/power failure. The
-unique `(ns, key)` index prevents two different rows for the same logical
-security.
-
-### Other places the key is stored
-
-| Location | Purpose | Authoritative for allocation? |
-|---|---|---|
-| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
-| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
-| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
-| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
-| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
-
-The `SecurityIDRecorder::work_db()` comment says a published security is also
-recorded in the security database, but that function does **not** write
-`securities.sqlite` in the current code. Full security definitions are persisted
-separately by `SecurityCache::flush_to_disk()` through
-`db::insert_or_replace()`.
-
-## CME MDP3 / OnixS pipeline in detail
-
-### 1. Construction and thread setup
-
-`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
-`MDProducer`. It constructs the recorder with:
-
-- a lookup callback that maps `ExchangeKey` through the CME index to the
-  handle's current Spark key; and
-- `Dispatcher::current()`, which drives a 200 ms recorder timer.
-
-In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
-dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
-then calls that thread's dispatcher `run()`. The older `md run onixs` command
-uses its command/dispatcher thread similarly.
-
-The OnixS feed engine has its own configured thread pool. Instrument and market
-data callbacks therefore originate on OnixS feed-engine threads, while recorder
-database work and final key assignment execute on the MDP3 dispatcher thread.
-
-| MDP3 stage | Executing thread | DB call? |
-|---|---|---|
-| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
-| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
-| Add CME-ID index entry | Same OnixS callback thread | No |
-| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
-| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
-| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
-
-### 2. Definition callback: exchange ID is set, Spark key is not
-
-For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
-
-```cpp
-fsec.numeric_id = def.securityId();
-```
-
-It also copies symbol/descriptor/venue and pricing fields. A newly constructed
-`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
-does not change it.
-
-For a spread, every leg starts with:
-
-```cpp
-leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
-leg.key = invalid_security_key;
-```
-
-Thus both the parent and legs initially carry exchange-routing identity, but no
-Spark identity.
-
-### 3. Shared-memory allocation and exchange indexing
-
-`MDP3Listener::add_security()` performs these operations in order:
-
-1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
-   cold entry and copies the security into the cold segment.
-2. Because the key is `0`, `MDProducer` queues the handle through
-   `SecurityIDRecorder::ensure_key()`.
-3. `_index.add_security(security.numeric_id, handle)` immediately creates the
-   CME-ID-to-handle mapping.
-4. The initial exchange status is written through the handle.
-
-This ordering means incoming books can already be routed by CME ID while the
-Spark key is pending. Shared-memory consumers cannot discover the security yet.
-Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
-when `key() != 0`, so a trade received in this short window updates the cold
-last-trade state but is not added to the keyed trade log.
-
-### 4. Recorder tick: spreads, handles, then database
-
-Every 200 ms the recorder calls, in this exact order:
-
-1. `work_spreads()` tries to fill previously unresolved spread leg keys by
-   asking `CMESecurityIndex` for each leg exchange ID.
-2. `work_handles()` drains new handles. Outrights are converted to an API
-   security and queued for DB lookup immediately. A spread without all leg keys
-   is retained in `pending_handle` and its leg pointers are registered in
-   `unresolved_legs`.
-3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
-
-Because spread work precedes new-handle work, a newly seen unresolved spread
-normally waits until at least the next timer tick to resolve legs. It waits
-longer if any outright leg has not yet received its Spark key. A spread's own
-key is not requested until every leg key is non-zero.
-
-### 5. Lookup or generation
-
-`work_db()` first queries `(ns, key)`:
-
-- Existing row: use its `rowid` as the key and invoke the callback.
-- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
-  rowid inside a transaction, the row object receives that ID, and the callback
-  runs after the transaction completes.
-
-New inserts are throttled to roughly 200 per timer pass (the current `> 200`
-check can admit 201) to avoid delaying heartbeats during startup. The queues are
-TBB concurrent queues, but all SQLite access is serialized onto the recorder's
-dispatcher thread. `check_access()` asserts this single-thread rule after the
-first work pass records the dispatcher thread ID.
-
-### 6. Final assignment and visibility
-
-The queued callback captures a copy of `MDProducerHandle` and calls
-`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
-then increments `security_count`. From this point:
-
-- shared-memory scans report the security;
-- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
-- trades and RFQs can be emitted with a non-zero key;
-- resolved spreads can copy this key into their leg structures.
-
-### 7. Sunday preload is a fast path
-
-For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
-definitions from `securities.sqlite`. It converts them with
-`set_security_key=true`, so both the security key and spread leg keys come from
-the serialized API security. `MDProducer::add_security()` calls `set_key()`
-immediately, and no recorder lookup is required for that preloaded handle. The
-CME exchange ID is still restored into `numeric_id`, allowing the normal CME
-index to be built.
-
-## Generalized/newer pipeline in detail
-
-### 1. Topology and ownership
-
-`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
-`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
-`MarketDataGateway`, and starts the configured providers. For each `(venue,
-source)` returned by a provider, the gateway creates an `MdSink` containing:
-
-- a venue-local `GeneralSecurityIndex`;
-- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
-- a `GeneralMarketDataListener` connecting the source to that producer/index.
-
-The source decides its callback thread. Some sources call the listener while
-starting; others own network or replay threads. The generalized listener calls
-the producer inline on that source thread. The process-wide `IDGenerator`
-serializes its own calls with a mutex, but the listener's transient
-`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
-expected to serialize security/event callbacks.
-
-| Generalized stage | Executing thread | DB call? |
-|---|---|---|
-| Build/parse domain definition | Source/adaptor thread | No |
-| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
-| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
-| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
-| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
-| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
-
-There are two important compatibility exceptions inside the provider framework:
-
-- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
-  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
-  legacy flow documented above.
-- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
-  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
-  producer/index. Its assignment is still synchronous and pre-populated, so it
-  resembles the newer ID flow even though it does not pass through
-  `GeneralMarketDataListener`.
-
-### 2. How a new ID is constructed
-
-For a normal live generalized adapter, ID creation happens **in the adapter,
-before** `GeneralMarketDataListener::on_securities()`:
-
-```text
-logical identity = (domain venue, domain security type, adapter-chosen name)
-Spark ID         = SQLite rowid for that identity in key_ids.sqlite
-```
-
-`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
-the enum prefixes:
-
-```text
-VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
-```
-
-It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
-row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
-next rowid. That integer is wrapped in `domain::SecurityID` and placed in
-`SecurityCommon::id`.
-
-The third identity component is adapter-specific, and this matters when trying
-to reproduce a mapping:
-
-| Source/adaptor family | Typical `name` sent to `IDGenerator` |
-|---|---|
-| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
-| Taifex | product/instrument ID string |
-| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
-| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
-| Emumba Deribit | extracted Emumba security descriptor |
-| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
-
-So “security name” alone is not a universal logical key. Reproduction requires
-the exact venue enum, exact security-type enum, and the exact adapter-specific
-name used at insertion time.
-
-### 3. What `GeneralMarketDataListener` does with that ID
-
-`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
-behavior is controlled by:
-
-```cpp
-const bool use_given_sec_key = true;
-```
-
-Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
-call the DB. It copies `security_common.id.value()` into its transient
-`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
-`_add_spot()` then calls `_copy_security_common()`, which sets:
-
-```cpp
-to.key = incoming_id;
-to.numeric_id = incoming_id;
-```
-
-This collapses the generalized source/correlation ID and Spark shared-memory
-key into the same number. The source's text `exchange_sec_id` remains on the
-domain security but is not copied into `FixedSecurity::numeric_id` by this
-path.
-
-`_commit_security()` then:
-
-1. calls `MDProducer::add_security()` with an already non-zero key;
-2. causes `MDProducerHandle::set_key()` to run immediately during that call;
-3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
-   number); and
-4. returns with the security already visible in shared memory.
-
-Because the sink's producer has no recorder, supplying `id == 0` would allocate
-the shared-memory entry but never assign or publish its key. Generalized live
-sources must therefore provide a valid non-zero ID before `on_securities()`.
-
-The listener contains an alternative branch for `use_given_sec_key == false`.
-That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
-sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
-also effectively unused. Turning the flag off is not a mechanical change: an
-adapter such as Databento allocates using numeric instrument ID while the
-listener branch would use `sec.desc()`, potentially creating a second key for
-the same instrument.
-
-There is also a spread failure edge case. `_find_sec(sec, true)` records the
-parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
-ID is absent, `_add_spread()` logs and returns without committing the spread. A
-later repeat of the parent definition is then classified as a duplicate and
-ignored, so definition ordering is a hard requirement in the current code.
-
-### 4. How later books, trades, and statistics find the security
-
-Every later domain event carries a `domain::SecurityID`:
-
-- `Book::security_id()`;
-- `TradeRecord::sec_id`; or
-- `SecurityMarketDataStatistics::security_id`.
-
-The listener first uses `cc_sec_id_map` to translate that event ID to the value
-stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
-`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
-but it still serves as the admission check that the definition was seen.
-
-Most high-throughput adapters avoid SQLite on market-data events:
-
-- Databento allocates on the definition and caches
-  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
-- Taifex keeps the resolved ID in its security-details mapper.
-- Other native subscriptions maintain comparable descriptor/ID maps.
-
-The Python bridge is a notable exception: `to_bts_security_()`,
-`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
-`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
-one SQLite `SELECT` for each converted Python definition, trade, or book. It
-returns the existing ID after the first insert, but it is still a DB call and
-takes the global `IDGenerator` mutex on every event.
-
-### 5. Persistence timing and latency
-
-The newer allocator is synchronous:
-
-- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
-  thread.
-- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
-  (SQLite autocommit), also on the caller's thread.
-- Listener publication: no further DB call under the current
-  `use_given_sec_key` setting.
-
-The ID therefore exists in `key_ids.sqlite` before the adapter sends the
-definition to the listener. There is no intentional 200 ms publication delay,
-but the source callback/startup path directly pays DB and mutex latency. Bulk
-definition discovery performs one lookup and possibly one insert per security;
-`IDGenerator` does not batch those operations.
-
-WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
-not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
-occasional file-system work. The impact is usually a startup/definition cost
-when adapters cache the result, but it becomes hot-path latency for code such as
-the current Python bridge that resolves every event through the DB.
-
-`GeneralMarketDataListener` itself does not persist the full contract definition
-to `securities.sqlite`; that remains the separate security-cache/metadata
-persistence path described earlier.
-
-## Comparison: MDP3 recorder vs generalized allocator
-
-| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
-|---|---|---|
-| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
-| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
-| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
-| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
-| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
-| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
-| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
-| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
-| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
-| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
-| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
-| Batching | Yes for new rows | No |
-| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
-
-Neither algorithm generates a UUID or a content hash. Both ultimately use an
-opaque, database-assigned SQLite rowid. It is normally increasing and looks like
-a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
-used to *look up* the row is logical; the resulting number is not derivable from
-that tuple.
-
-## Can another system reconstruct the same keys?
-
-Not from `(venue, type, name)` alone. Another system can reconstruct the
-**logical lookup tuple**, but it cannot calculate the numeric Spark key because
-the key depends on the historical insertion order of every row in
-`key_ids.sqlite`.
-
-For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
-row and receive different IDs if other securities were inserted before it on
-one system. They are sequential opaque IDs, not deterministic hashes and not
-UUIDs.
-
-Ways to preserve cross-system identity are:
-
-1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
-   identical. This preserves existing numeric keys but requires coordinated
-   allocation for new ones.
-2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
-   import rows with their original rowids before either system allocates new
-   securities.
-3. Treat a stable external identity (venue + type + exchange instrument ID, or
-   another canonical instrument identifier) as the cross-system key and carry a
-   translation table to each system's local Spark key.
-4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
-   be computed independently, but collision handling, descriptor
-   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
-   with orders/recordings/databases all need explicit design.
-
-Copying only `securities.sqlite` is insufficient as an allocator strategy. It
-contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
-the mapping used to find or allocate those IDs. Conversely, copying only
-`key_ids.sqlite` preserves the numbers but not all contract metadata.
-
-## Other assignment paths in `sparkd`
-
-The two main pipelines reduce to four ways a producer handle can acquire a key:
-
-1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
-   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
-   Sunday preload, relay playback, and other replay paths use this mechanism.
-2. **Recorder-backed live input.** The input key is zero and the producer owns a
-   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
-   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
-   `numeric_id` and otherwise follows the same recorder/index design.
-3. **Service assignment.** `SyntheticSecurityService` calls
-   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
-   callback receives a copied `api::Security` with its top-level `id` set, then
-   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
-   rowid allocator.
-4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
-   never becomes public. This is not a valid production assignment path, but it
-   is the failure mode if a generalized source omits its ID.
-
-Relay and replay deserve care when reasoning about persistence:
-
-- The generalized `MDReplayer` forwards the recorded domain security ID and
-  emits later books/statistics with the recorded `sec_key`, so the listener's
-  current use-given mode preserves the recording's numeric identity.
-- The legacy relay producer converts a recorded `api::Security` with
-  `set_security_key=true`, so its shared-memory publication also preserves the
-  embedded key.
-- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
-  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
-  separate mapping population step, not a deterministic derivation of the key.
-
-## Operational implications
-
-- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
-  definition continuity matter.
-- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
-  another host, even when all instruments are identical.
-- Before moving a feed between adapters, compare the exact `name` each adapter
-  supplies to `IDGenerator`. Matching venue/type is insufficient.
-- Cache generated IDs in the source after definition processing. Avoid calling
-  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
-  this guideline.
-- Keep generated rowids below the 32-bit shared-memory ceiling until the
-  `security_key` ABI is widened or explicit bounds checking is added to the
-  generalized path.
-- Ensure spread legs are announced before generalized spread parents.
-- Treat `synchronous=0` as a durability decision. A copied DB should be taken
-  with SQLite-aware backup/checkpoint handling rather than by casually copying
-  active WAL files.
-
-Useful inspection queries (run against the configured DB path) are:
-
-```sql
-SELECT rowid AS spark_key, ns, key
-FROM store
-ORDER BY rowid;
-
-SELECT rowid AS spark_key, ns, key
-FROM store
-WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
-```
-
-## Source map
-
-- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
-  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
-  and spread-leg resolution.
-- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
-  synchronous generalized lookup/allocation and process-wide mutex.
-- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
-  location, schema, WAL/synchronous pragmas, individual inserts, and batch
-  inserts.
-- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
-  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
-  shared-memory allocation, recorder handoff, and ordered publication.
-- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
-  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
-  widths and hot/cold storage fields.
-- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
-  and
-  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
-  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
-- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
-  producer/index insertion order.
-- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
-  current dedicated MDP3 dispatcher-thread ownership.
-- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
-  and
-  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
-  generalized source/sink topology, incoming-ID mapping, fixed-security
-  conversion, and publication.
-- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
-  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
-  and
-  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
-  representative adapter-side allocation and caching behavior.
-- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
-  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
-  persistence of full security definitions.
-- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
-  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
-  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
-  translation followed by Spark-keyed market-data consumption and MBO access.
-- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
-  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
-  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
-  representative downstream use of Spark security keys.
-- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
-  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
-  ID reuse.

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -1,44 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-RUN apt-get update && apt-get install -y tzdata
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet8
-
-RUN apt-get update && apt-get install -y gdb
-RUN apt-get update && apt-get install -y fish
-RUN apt-get update && apt-get install -y vim
-RUN apt-get update && apt-get install -y less
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts

docker/images/bluetradingsystems/linux_shell/Dockerfile

@@ -1,11 +0,0 @@
-FROM ubuntu:24.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y \
-  fish \
-  less \
-  vim
-
-USER bts
-WORKDIR /home/bts

requirements.txt

@@ -16,16 +16,11 @@ jmespath==1.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
-protobuf==6.33.4
+protobuf==4.21.12
 pybars3==0.9.7
-pybind11==3.0.1
-pybind11-stubgen==2.5.5
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
 python-dateutil==2.9.0.post0
@@ -36,7 +31,6 @@ six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
 Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0

spark_client.sln

@@ -1,306 +1,103 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
 		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|Any CPU = Release|Any CPU
+		MinSizeRel|x64 = MinSizeRel|x64
 		Release|x64 = Release|x64
-		Release|x86 = Release|x86
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionGrammar.y

@@ -8,6 +8,7 @@
    public int Integer;
    public double Real;
    public string Identifier;
+   public DateTime DateTime; 
    public ImmutableList<IArithmeticExpression> Arguments;
    public ImmutableList<Assignment> Assignments;
    public IArithmeticExpression Argument;

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -11,20 +11,10 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ArithmeticExpression">
     <Scanner Include="ArithmeticExpressionTokens.lex" />

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -5,5 +5,9 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
-  
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
 </Project>

src/DataStreamSuite/BlueRegistry/DoubleRegistryHandler.cs

@@ -1,6 +1,4 @@
 using System;
-using System.Runtime.Versioning;
-
 using Microsoft.Win32;
 
 namespace BCG
@@ -12,7 +10,6 @@ namespace BCG
          rk.SetValue (name, value.ToString ());
       }
 
-      [SupportedOSPlatform("windows")]
       public static double getValue (RegistryKey rk, string name)
       {
          object v = rk.GetValue (name);

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,80 +6,23 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var obj1 = value1.GetValue(env);
-      var obj2 = value2.GetValue(env);
-      if (obj1 is null) {
-         if (obj2 is null) {
-            return 0;
-         }
-         return -1;
-      }
-      if (obj2 is null) {
-         return 1;
-      }
-      return compareObjs(obj1, obj2);
-   }
-
-   static int compareObjs(object obj1, object obj2) {
+      var cmp1 = value1.Cast<IComparable>();
+      var cmp2 = value2.Cast<IComparable>();
       try {
-         var typ1 = obj1.GetType();
-         var typ2 = obj2.GetType();
-         bool areSameTypes = typ1 == typ2;
-         if (areSameTypes) {
-            return ((IComparable)obj1).CompareTo(obj2);
-         }
-         if (typ1.IsNumeric()) {
-            return tryCompareNumeric(obj1, obj2);
-         } else if (typ2.IsNumeric()) {
-            return -tryCompareNumeric(obj2, obj1);
-         } else if (typ1.IsEnum) {
-            if (typ2 == typeof(string)) {
-               return -((string)obj2).CompareTo(obj1.ToString());
-            }
-         } else if (typ1 == typeof(string)) {
-            return ((string)obj1).CompareTo(obj2.ToString());
-         } else if (typ1 == typeof(DateTime)) {
-            if (typ2 == typeof(TimeSpan)) {
-               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
-            }
-         } else if (typ1 == typeof(TimeSpan)) {
-            if (typ2 == typeof(DateTime)) {
-               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
-            }
-         }
+         var v1 = cmp1.GetValue(env);
+         var v2 = cmp2.GetValue(env);
+         return v1.GetType().IsNumeric() ?
+            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
-      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
-   }
-
-   static int tryCompareNumeric(object numericObj1, object obj2) {
-      var typ2 = obj2.GetType();
-      if (typ2.IsNumeric()) {
-         return new NumericComparer().Compare(numericObj1, obj2);
-      } else if (typ2.IsEnum) {
-         var undTyp2 = typ2.GetEnumUnderlyingType();
-         if (undTyp2 == typeof(byte)) {
-            return new NumericComparer().Compare(numericObj1, (byte)obj2);
-         }
-         if (undTyp2 == typeof(short)) {
-            return new NumericComparer().Compare(numericObj1, (short)obj2);
-         }
-         if (undTyp2 == typeof(int)) {
-            return new NumericComparer().Compare(numericObj1, (int)obj2);
-         }
-         if (undTyp2 == typeof(long)) {
-            return new NumericComparer().Compare(numericObj1, (long)obj2);
-         }
-      }
-      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpression.cs

@@ -1,11 +1,19 @@
 namespace Bts.BooleanExpressions;
 
+using bts;
+
 class BooleanExpression : IBooleanExpression {
    readonly Func<IDictionary<string, object?>, bool> evaluate;
 
    public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
       this.evaluate = evaluate;
 
-   public bool Evaluate(IDictionary<string, object?> env) =>
-      evaluate(env);
+   public bool Evaluate(IDictionary<string, object?> env) {
+      try {
+         return evaluate(env);
+      } catch (Exception ex) {
+         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
+         throw;
+      }
+   }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -47,6 +47,11 @@ IsOfType                 '/'
 
 %%
 
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {BooleanConstant} {
    yylval.Boolean = yytext.ToLower() switch {
       "true" => true,
@@ -57,11 +62,6 @@ IsOfType                 '/'
    return (int)Token.BOOLEAN_CONSTANT;
 }
 
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {NullConstant} {
    yylval.Null = DBNull.Value;
    return (int)Token.NULL_CONSTANT;

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -11,20 +11,10 @@
     <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
   
   <ItemGroup>
     <Scanner Include="BooleanExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -10,20 +10,11 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnExpression">
     <Scanner Include="ColumnExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -10,20 +10,11 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnSortExpression">
     <Scanner Include="ColumnSortExpressionTokens.lex" />

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -7,21 +7,15 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnMapExpression">
     <Scanner Include="ColumnMapExpressionTokens.lex" />

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
-      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
-         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CSVReaderStream.cs

@@ -1,18 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-
-   using BCG;
-
-   [DataStreamUserRespresentation("CSVReaderStream")]
-   public class CSVReaderStream : ViewStream {
-      public CSVReaderStream(string file_name)
-         : this(file_name, false) { }
-      public CSVReaderStream(string file_name, bool guess_schema) {
-         ViewStream rs = new BCG.CSVReaderStream(file_name);
-         if (guess_schema) {
-            rs = new SchemaGuesserStream(rs);
-         }
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CSVSaverStream.cs

@@ -1,13 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("CSVSaverStream")]
-   public class CSVSaverStream: ViewStream {
-      public CSVSaverStream(ViewStream parent_str, string file_name) {
-         BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,43 +0,0 @@
-namespace BCG {
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
-      IDataPacket? myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
-      private int myId = 0;
-      private ViewStream myVS;
-
-      public CountStream(ViewStream vs) {
-         myVS = vs;
-      }
-
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
-
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
-            myLastFiredDataPacket =
-               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
-                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  myLastFiredDataPacket.newRows)
-            );
-      }
-
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow(new BlueRow(mySchema, id, ((IDataPacket)dp).newRows.Count));
-         return retVal;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -1,40 +0,0 @@
-namespace bts.BlueOrb {
-   using bts;
-
-   using BCG;
-
-   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_year");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            rv.setValue($"{inp_col}_year", dt.Date.Year);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,164 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using Bts.DataStreamCore;
-
-   using System;
-   using System.Text.RegularExpressions;
-
-   [DataStreamUserRespresentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      public class RegExpFilter : IStreamFilter {
-         private Regex[] myRegExps;
-         private string[] myCols;
-
-         public RegExpFilter(string col, string regExp)
-            : this(new string[] { col }, new string[] { regExp }) { }
-
-         public RegExpFilter(string[] cols, string[] regExps) {
-            myCols = cols;
-            myRegExps = new Regex[regExps.Length];
-            for (int i = 0; i < regExps.Length; i++)
-               myRegExps[i] = new Regex(regExps[i]);
-         }
-
-         #region IStreamFilter Members
-         public bool filter(IBlueRow dr) {
-            for (int i = 0; i < myCols.Length; i++) {
-               var val = dr[myCols[i]];
-               var str = val is DBNull
-                  ? string.Empty
-                  : (val?.ToString() ?? string.Empty);
-               if (!myRegExps[i].IsMatch(str)) {
-                  return false;
-               }
-            }
-            return true;
-         }
-         #endregion
-      }
-      BlueTable? myLastFiredRows;
-      IBlueRowCollection? myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-
-      #region constructors
-      public delegate bool OnFilter(IBlueRow r);
-      private class OnFilterWrapper : IStreamFilter {
-         public OnFilterWrapper(OnFilter f) { this.f = f; }
-
-         public bool filter(IBlueRow dr) { return f(dr); }
-
-         private OnFilter f;
-      }
-      public FilterStream(ViewStream vs, OnFilter f)
-         : this(vs, new OnFilterWrapper(f)) { }
-
-
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
-         myParentStream = vs;
-         myFilter = filter;
-         myFilterString = filter.ToString() ?? string.Empty;
-         if (filter is IDynamicStreamFilter dynFilter) {
-            dynFilter.onFilterChange += myHandleFilterChange;
-         }
-      }
-
-      public FilterStream(ViewStream vs, string filterExpression)
-         : this(vs, new BooleanExpressionFilter(filterExpression)) {
-         myFilterString = filterExpression;
-      }
-
-      public FilterStream(ViewStream vs, string col, string regExp)
-         : this(vs, new RegExpFilter(col, regExp)) {
-         myFilterString = col + " , " + regExp;
-      }
-
-      public FilterStream(ViewStream vs)
-         : this(vs, new BinaryFilter(true)) {
-         myFilterString = true.ToString();
-      }
-      #endregion
-
-      #region overridden methods
-      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
-
-      protected override void myOnLastSubscriber() {
-         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
-      }
-
-      protected override void myFireEvent(object o) {
-         var dp = (IDataPacket)o;
-         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
-            base.myFireEvent(o);
-      }
-      #endregion
-
-      private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         IDataPacket arrivedPacket = (IDataPacket)newValue;
-         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
-            foreach (IBlueRow r in arrivedPacket.deletedRows) {
-               if (myLastFiredRows.containsID(r.Id())) {
-                  myLastFiredRows.deleteRow(r);
-                  deletedRows.Value.addRow(r);
-               }
-            }
-            var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
-            foreach (IBlueRow r in arrivedPacket.addedRows) {
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  addedRows.Value.addRow(r);
-               }
-            }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.Schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
-      }
-
-      public IStreamFilter filter {
-         set {
-            {
-               myFilter = value;
-               if (numberOfSubscribers > 0) {
-                  if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
-                  }
-               }
-            }
-         }
-         get {            
-               return myFilter;
-         }
-      }
-
-      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
-
-      public override string ToString() {
-         return GetType().ToString() + " " + myFilterString;
-      }
-
-      private void myHandleFilterChange() {
-         filter = myFilter;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -1,142 +0,0 @@
-using BCG;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("GroupByStream")]
-   public class GroupByStream : ViewStream {
-
-      public interface INonIncGroupByEvaluator {
-         // grouped_rows
-         IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[]? get_group_by_cols();
-      }
-
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]?> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-
-         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable? rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               if (rv != null)
-                  rv.addRows(out_rows);
-            }
-            return rv;
-         }
-
-         public string[]? get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
-      ViewStream? _main_stream;
-      INonIncGroupByEvaluator? _groupby_func;
-      BlueRowHasher? _row_hasher;
-      string[]? _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      IDisposable? subscription;
-
-      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
-         _main_stream = main_stream;
-         _groupby_func = groupby_func;
-         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
-      }
-
-      public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
-      : this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
-
-      // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            // this si where the group by aggregation happens
-            IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            if (rv == null)
-               return new List<IBlueRow>();
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-
-         string[]? groupby_cols_func() {
-            if (string.IsNullOrEmpty(groupby_cols)) {
-               return null;
-            }
-            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
-         }
-      }
-      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-      protected override void myOnFirstSubscriber() {
-         Log.isnull(subscription);
-         subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
-      }
-
-      protected override void myOnLastSubscriber() {
-         subscription?.Dispose();
-         subscription = null;
-      }
-
-      private void _main_stream_onchange(object nv) {
-         Log.notnull(_groupby_func);
-         var dp = (IDataPacket)nv;
-         ///grouping together
-         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
-         /// reducing
-         _timer.Restart();
-         IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
-         if (total_state == null)
-            // mayeb returna n empty table ?
-            return;
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
-            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         var deletedRows = myLastValue is null
-            ? new BlueTable(total_state.schema)
-            : ((IDataPacket)myLastValue).newRows;
-         myFireEvent(new DataPacket(total_state, total_state, deletedRows));
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,204 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using Bts.DataStream;
-
-   using Spark.Core.Diagnostics;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Diagnostics.CodeAnalysis;
-   using System.Linq;
-
-   using RowHash = System.Int32;
-
-   [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
-         // initialized in cons
-         readonly string[] _join_col_names;
-         private readonly BlueRowHasher _hasher;
-         // initialized on first udpate         
-         IBlueTableSchema? _schema;
-         int[]? _join_cols;
-         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
-            _hasher = row_hasher;
-         }
-
-         [NotNullIfNotNull(nameof(_schema))]
-         public IBlueTableSchema? schema => _schema;
-
-         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
-
-         [MemberNotNull(nameof(_schema))]
-         public IBlueTableSchema RequireSchema() =>
-            ExceptionAssert.MemberNotNull(_schema);
-
-         public List<IBlueRow>? get_matching_rows(int hash) =>
-            _hash_row_map.GetValueOrDefault(hash);
-
-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) {
-               return 0;
-            }
-            return _hasher.get_hash(r, _join_cols);
-         }
-
-         public void update_data(IDataPacket dp) {
-            if (!ready) {
-               _schema = dp.newRows.schema;
-               _join_cols = _schema.ResolveIndices(_join_col_names);
-            }
-            _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
-               int hash = _get_hash(r);
-               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
-                  hash_rows = new List<IBlueRow>();
-                  _hash_row_map.Add(hash, hash_rows);
-               }
-               hash_rows.Add(r);
-            }
-         }
-
-         [MemberNotNullWhen(true, nameof(_schema))]
-         public bool ready {
-            get {
-               return _schema != null;
-            }
-         }
-
-      }
-
-      private ViewStream _ls, _rs;
-      private JoinBookkeepingData _ldata, _rdata;
-      IBlueTableSchema? _schema;
-
-      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
-
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
-         _ls = ls;
-         _rs = rs;
-         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count == rj_col_list.Count);
-         int n_cols = lj_col_list.Count;
-         BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _ls.onChange += _ls_onchange;
-         _rs.onChange += _rs_onchange;
-      }
-
-      private void _ls_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         _ldata.update_data(dp);
-         _form_and_fire();
-      }
-
-      private void _rs_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         _rdata.update_data(dp);
-         _form_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         _ls.onChange -= _ls_onchange;
-         _rs.onChange -= _rs_onchange;
-      }
-
-      [MemberNotNull(nameof(_schema))]
-      private void _build_schema() {
-         if (_schema != null)
-            return;
-         var lschema = Log.notnull(_ldata.schema);
-         var rschema = Log.notnull(_rdata.schema);
-         var cis = new List<ColumnInfo> {
-            new("id", 0, TypeCode.Int32)
-         };
-         foreach (ColumnInfo ci in lschema.columnInfo) {
-            string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         foreach (ColumnInfo ci in rschema.columnInfo) {
-            string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
-      }
-
-      private void _form_and_fire() {
-         if (!_ldata.ready || !_rdata.ready)
-            return;
-         _build_schema();
-         List<object[]> res_rows = new List<object[]>();
-
-         // this is where the join happens
-         foreach (var kv in _ldata.rows) {
-            RowHash hash = kv.Key;
-            List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
-
-            foreach (IBlueRow lr in l_rows) {
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
-                     object[] rowvals = build_row_array(res_rows.Count, lr);
-                     copy_row(rowvals, r, false);
-                     res_rows.Add(rowvals);
-                  }
-               } else {
-                  object[] rowvals = build_row_array(res_rows.Count, lr);
-                  res_rows.Add(rowvals);
-               }
-            }
-         }
-         BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
-            rv.addRow(new BlueRow(_schema, rval));
-         }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
-         myFireEvent(dp);
-
-         object[] build_row_array(int id, IBlueRow l_row) {
-            object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         }
-
-         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.RequireSchema().columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         }
-      }
-   }
-
-   [DataStreamUserRespresentation("FullJoinStream")]
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/BlueOrb/JsonSaverStream.cs

@@ -1,28 +0,0 @@
-//namespace bts.Blueorb {
-//   using BCG;
-
-//   public class JsonSaverStream : ViewStream {
-//      private ViewStream _stream;
-//      private string _file_name;
-//      public JsonSaverStream(ViewStream vs, string file_name) {
-//         _stream = vs;
-//         _file_name = file_name;
-//      }
-
-//      protected override void myOnFirstSubscriber() {
-//         _stream.onChange += _on_data;
-//      }
-
-//      private void _on_data(object dp_obj) {
-//         if (dp_obj == null) {
-//            return;
-//         }
-//         DataPacket dp = dp_obj as DataPacket;
-//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
-//      }
-
-//      protected override void myOnLastSubscriber() {
-//         _stream.onChange -= _on_data;
-//      }
-//   }
-//}

src/DataStreamSuite/DataStream/BlueOrb/LiteralStream.cs

@@ -1,91 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-using Amazon.Runtime.Internal.Util;
-
-using BCG;
-using BCG.Utilities;
-
-using bts;
-
-namespace bts.BlueOrb{
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      // col_name: col_value ... ; col_name: col_value
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
-
-
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
-         int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
-            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
-            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
-            n++;
-         }
-
-
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
-         n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
-                  continue;
-               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
-               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
-            }
-
-         mySetRealSource(new BlueRowCollectionStaticStream(t));
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -1,89 +0,0 @@
-using System;
-
-using BCG.Utilities.General;
-using BCG;
-using bts;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("MillisecondStream")]
-   public class MillisecondStream : ViewStream, IWorker {
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private readonly TimeSpan _period;
-      private readonly int _size;
-      private readonly ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema;
-      private bts.CircularBufferTable _table;
-
-      public MillisecondStream(int interval_msecs, int size)
-      : this(ViewStreamDatabase.instance, interval_msecs, size) {
-      }
-
-      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-         _vsd = vsd;
-         _period = TimeSpan.FromMilliseconds(interval_msecs);
-         _size = size;
-         _schema = BlueDataUtility.form_schema("Milliseconds",
-               Tuple.Create("id", TypeCode.Int32),
-               Tuple.Create("time", TypeCode.DateTime),
-               Tuple.Create("display_time", TypeCode.String));
-         _table = new CircularBufferTable(_schema, _size);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private int _id = 0;
-      public void doWork() {
-         if (DateTime.Now - _last_fire_time >= _period) {
-            _last_fire_time = DateTime.Now;
-            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
-            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
-   }
-   internal class MSecondStream : TimeUnitStream {
-      public MSecondStream(int intervalInms, int size)
-         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
-
-      protected override DateTime myRoundTime(DateTime dt) { return dt; }
-   }
-
-   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
-      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
-                                                               new ColumnInfo("id", 0, typeof(int)),
-                                                               new ColumnInfo("time", 1, typeof(string)));
-      public DisplayableMillisecondStream(int intervalInMs, int sz) {
-         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
-      }
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         DateTime time = row.get_datetime("time");
-         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-         return new BlueRow(mySchema, row["id"], time_txt);
-      }
-
-      #endregion
-
-      private string myToString(DateTime dt) {
-         string sep = ":";
-         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -1,168 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-   }
-   public class MultiRowOpHelper {
-      // input + op
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse_ops(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count == 0) {
-            return null;
-         }
-         IBlueRow ir = rows.ElementAt(start_idx);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         // out_col vs inp_cal + type
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-            or.setValue("id", ir["id"]);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.Median:
-               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
-            //case MultiRowOp.Mode:
-            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            case MultiRowOp.EdgeDiff:
-               return col_vals.First() - col_vals.Last();
-            case MultiRowOp.EdgeDiffPercentage: {
-                  double first = col_vals.First();
-                  double last = col_vals.Last();
-                  return ((first - last) / last) * 100.0;
-               }
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -1,40 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
-   public class MultiRowRollingMathStream : ViewStream {
-      private ViewStream _sorted_parent_str;
-      private MultiRowOpHelper _op;
-      private int _rolling_window_sz;
-      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
-         _sorted_parent_str = new SortStream(parent_str, sort_exp);
-         _op = MultiRowOpHelper.parse_ops(op_txt);
-         _rolling_window_sz = rolling_window_sz;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _sorted_parent_str.onChange += _parent_str_onchange;
-      }      
-
-      protected override void myOnLastSubscriber() {
-         _sorted_parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         var rows = dp.newRows.to_list();
-         BlueTable? out_table = null;
-         for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
-               ?? throw new InvalidOperationException("group_by_compute returned null row");
-            if (out_table == null)
-               out_table = new BlueTable(out_r.schema);
-            out_table.addRow(out_r);
-         }            
-         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,190 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using Spark.Core.Diagnostics;
-
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   #region mappings
-   internal class ProjectData {
-      // schema from the base stream
-      public readonly IBlueTableSchema in_schema;
-      // this is the projected schema
-      public readonly IBlueTableSchema out_schema;
-      // projected cols to the input cols, for cols who's names are changed
-      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
-      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
-                  IDictionary<string, string> mapped_to_orig_col_map_) {
-         in_schema = in_schema_;
-         out_schema = out_schema_;
-         mapped_to_orig_col_map = mapped_to_orig_col_map_;
-      }
-   }
-   #endregion
-
-
-#region Row with mappings build in
-   internal class NewProjectRow : IBlueRow {
-      #region IBlueRow members
-      private ProjectData _project_data;
-      private IBlueRow _row;
-      public NewProjectRow(ProjectData project_data, IBlueRow row) {
-         _project_data = project_data;
-         _row = row;
-      }
-      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
-      #endregion
-
-      #region ICollection members
-      public int Count { get { return schema.columnNames.Length; } }
-
-      public bool IsSynchronized { get { return false; } }
-
-      public object SyncRoot { get { return this; } }
-
-      public void CopyTo(Array array, int index) {
-         int i = index;
-         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
-      }
-      #endregion
-
-      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
-
-      public bool IsReadOnly { get { return true; } }
-
-      public object? this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            return this[(int)key];
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public object? this[int index] {
-         get {
-            return this[schema.columnInfo[index].colName];
-         }
-      }
-
-      public object? this[string col_name] {
-         get {
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
-               return _row[mapped_to_col];
-            }
-            return _row[col_name];
-         }
-      }
-
-      public ICollection Keys { get { return schema.columnNames; } }
-
-      public ICollection Values {
-         get {
-            object[] retVal = new object[schema.columnNames.Length];
-            CopyTo(retVal, 0);
-            return retVal;
-         }
-      }
-
-      public void Add(object key, object? value) =>
-         throw new NotSupportedException();
-
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public bool Contains(object key) { return schema[(string)key] != null; }
-
-      /// <summary>
-      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
-      /// iterate through the values, typecast this into an IEnumerable
-      /// </summary>
-      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-      #endregion
-   }
-   #endregion
-
-   [DataStreamUserRespresentation("ProjectStream")]
-   public class NewProjectStream : ViewStream, IStreamMapper {
-
-      private ViewStream _stream;
-      private string _project_expression;
-      ProjectData? _project_data;
-      //public ProjectData 
-      public NewProjectStream(ViewStream vs, string project_expression) {
-         _stream = vs;
-         _project_expression = project_expression;
-         _stream = new MapStream(vs, this);
-         mySetRealSource(_stream);
-      }
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-         _project_data = _create_project_data(input_schema, _project_expression);
-         return _project_data.out_schema;
-      }
-
-      public IBlueRow map(IBlueRow row) =>
-         new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
-
-      // this is where most of the logic is, in terms of translating cols
-      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
-                                                      string project_expression) {
-         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ', ',');
-         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // modifications (6/30/2025):
-         // only the columns represented here, either mapped
-         // or not, will show
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa b") -> aa b
-         foreach (string o_p_tok in project_toks) {
-            string p_tok = o_p_tok.Trim();
-            if (p_tok.Contains(":")) {
-               // explicit mapping
-               string[] map_toks = p_tok.Split(":");
-               string orig_col = map_toks[0];
-               string mapped_col = map_toks[1];
-               orig_to_mapped.Add(orig_col, mapped_col);  
-            } else {
-               // this is a col_name explicitely stated
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
-            }
-         }
-
-         // make the new schema
-         List<ColumnInfo> new_cols = new List<ColumnInfo>();
-         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;           
-            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
-               string mapped_name = orig_to_mapped[ci.colName];
-               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-            }
-         }
-         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
-         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
-      }
-
-   
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs

@@ -1,128 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System.IO;
-   using System.Diagnostics;
-   using System;
-
-   [DataStreamUserRespresentation("PythonDataframeStream")]
-   public class PythonDataframeStream : ViewStream {
-      private ViewStream _parent_str;
-      private readonly string _in_csv_file_name;
-      private readonly string _out_csv_filename;
-      private readonly string _transform_function;
-      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
-         var guid = System.Guid.NewGuid();
-         _in_csv_file_name = $"pdfs_in_{guid}.csv";
-         _out_csv_filename = $"pdfs_out_{guid}.csv";
-         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
-         _transform_function = transform_function;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _parent_str_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         Log.assert(dp_obj != null);
-         // the file _in_csv_file_name is now populated.
-         // call py_func_wrapper_main and pass this file name as well
-         // as the output file name
-         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
-         // transform function, which returns the tranformed df. py_func_wrapper_main then
-         // writes it into the _out_csv_filename file.
-         // now this function should read _out_csv_filename into a dp
-         // and fire it
-         _compute_and_write();
-         //ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
-         ViewStream rs = new CSVReaderStream(_out_csv_filename);
-         IBlueRowCollection nrows = null;
-         rs.onChange += (dp_obj) => {
-            nrows = (dp_obj as DataPacket).newRows;
-         };
-         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      private void _compute_and_write() {
-         File.WriteAllText("user_code.py", _transform_function);
-         // wrapper function
-         string py_func_wrapper_main = @"
-import sys
-from user_code import transform
-import pandas as pd
-
-if __name__ == '__main__':
-   input_file_name = sys.argv[1]
-   output_file_name = sys.argv[2]
-# 1. load input_file_name into df
-# 2. do some test calc
-# 3. write the output of 2 into output_file_name
-   df = pd.read_csv(input_file_name)   
-   df_out = transform(df)
-   df_out.to_csv(output_file_name, index=False)";
-
-
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-      }
-
-      #region Unused test code
-      private void _simple_test() {
-
-         // actual function that needs to be called ////////////////////////
-         string py_real_function = @"
-def test(x, out_path):
-   rv = x * 10
-   with open(out_path, 'w') as f:
-      f.write(str(rv))
-   return rv";
-         // the function is in a string object.
-         File.WriteAllText("user_code.py", py_real_function);
-
-
-         // runner app (top level main function) //////////////////////////////
-         string py_func_wrapper_main = @"
-import sys
-from user_code import test
-
-if __name__ == '__main__':
-   arg = int(sys.argv[1])
-   out_path = sys.argv[2]
-   rv = test(arg, out_path)
-   rv = rv + 12
-   with open(out_path, 'w') as f:to \
-      f.write(str(rv))";
-   
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         string outputPath = "py_out.txt";
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py 4 {outputPath}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-         // Now read the output written by the function
-         string result = File.ReadAllText(outputPath);
-         Console.WriteLine(result); // Should print "6"
-
-      }
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SchemaGuesserStream.cs

@@ -1,30 +0,0 @@
-using BCG;
-
-namespace bts.BlueOrb {
-   public class SchemaGuesserStream : ViewStream {
-
-      private IBlueRowCollection _rows = null;
-      private ViewStream _vs;
-      public SchemaGuesserStream(ViewStream vs) {
-         _vs = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vs.onChange += _on_data;
-      }
-      protected override void myOnLastSubscriber() {
-         _vs.onChange -= _on_data;
-      }
-
-      private void _on_data(object obj) {
-         DataPacket dp = obj as DataPacket;
-         if (_rows == null) {
-            _rows = BlueDataUtility.get_typed_table(dp.newRows);
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SingleRowMathStream.cs

@@ -1,67 +0,0 @@
-using Bts.ArithmeticExpressions;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG;
-
-namespace bts.BlueOrb {
-
-   [DataStreamUserRespresentation("SingleRowMathStream")]
-   public class MathStream : ViewStream, IStreamMapper {
-      private IArithmeticProgram myExp;
-      private BooleanExpressionFilter _bool;
-      List<string> myExtraColumns = new List<string>();
-      BlueTableSchema mySchema;
-
-
-      public MathStream(ViewStream vs, string mathExpression)
-         : this(vs, mathExpression, string.Empty) {
-      }
-
-      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
-         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
-            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
-         myExtraColumns = myExp.OutputIdentifiers.ToList();
-         if (!string.IsNullOrEmpty(boolean_expression))
-            _bool = new BooleanExpressionFilter(boolean_expression);
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
-            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
-            foreach (ColumnInfo ci in inputSchema.columnInfo)
-               colIdx[ci.colName] = ci;
-
-            foreach (string ncn in myExtraColumns) {
-               ColumnInfo newC;
-               if (colIdx.ContainsKey(ncn))
-                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
-               else
-                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
-               colIdx[ncn] = newC;
-            }
-            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         MutableBlueRow result = new MutableBlueRow(mySchema, row);
-         if (_bool == null || _bool.filter(row)) {
-            myExp.Evaluate(result.AsDictionary());
-         }
-         return result;
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs

@@ -1,17 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SortStream")]
-   public class SortStream: ViewStream {
-      public SortStream (ViewStream parent_stream, string sort_expr) {
-         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TimeBasedRollingFilterStream.cs

@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
-   public class TimeBasedRollingFilterStream : ViewStream {
-      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
-         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TopStream.cs

@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("TopStream")]
-   public class TopStream : ViewStream { 
-      public TopStream(ViewStream vs, int n_rows, string sort_col) {
-         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -1,17 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-   using System.Collections.Generic;
-
-   using mbts;
-   using BCG;
-
-   // [DataStreamUserRespresentation("UnionStream")]
-   [DataStreamUserRespresentation("AppendStream")]
-   public class UnionStream : ViewStream {
-      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
-         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -5,9 +5,19 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
+    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
@@ -16,27 +26,4 @@
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-        <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
-        <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-        <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-        <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
-        <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ArithmeticExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="BaseUtilities" Version="$(SparkVersion)" />
-        <PackageReference Include="BooleanExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.ColumnExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.ColumnSortExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="DataStreamExpressions" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
-
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,217 +2,25 @@ namespace Bts.DataStream;
 
 using BCG;
 
-using bts.api.core;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using Microsoft.Extensions.Logging;
-
 using Spark.Core.Disposables;
-using Spark.Core.Threading;
 
 using System;
-using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
 
-   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<IDataPacket> observer) {
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
          source.onChange += OnChange;
          return Disposable.Create(() => source.onChange -= OnChange);
 
          void OnChange(object? newValue) =>
-            observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
-               $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
-
-   // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-   //    source.Select(dp => {
-   //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-   //       return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
-
-   //       IBlueRowCollection<T> CastRows(ICollection rows) =>
-   //          new BlueTableSelectorWrapper<T>(
-   //             rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-   //    });
-
-   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
-      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
-      source.AsTables<T, int>(valueSelector, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
-      source.AsTables<T, int>(valueSelector, keySelector);
-
-   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
-      rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.Empty(keySelector), (res, dp) => {
-         var bd = res.ToBuilder();
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         foreach (IBlueRow r in addedRows) {
-            bd.AddOrUpdate(valueSelector(r));
-         }
-         foreach (IBlueRow r in deletedRows) {
-            bd.Remove(rowKeySelector(r));
-         }
-         return bd.ToImmutable();
-      });
-   }
-
-   public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
-      source.Scan(
-         (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
-         (res, table) => {
-            var resTable = res.Table;
-            if (resTable is null) {
-               if (table.Index.Count == 0) {
-                  return (null, null);
-               }
-               resTable = new(table.Index.Values.First().schema);
-            }
-            var bd = resTable.ToDataPacketBuilder();
-            foreach (var r in table.RemovedItems) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.AddedItems) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacket(out var newRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0))
-         .Select(t => t.Dp!);
-
-   public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.Scan(
-         (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
-         (res, table) => {
-            var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.RemovedItems) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.AddedItems) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacket(out var newRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)
-         .Select(t => t.Dp);
-
-   public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
-      return Observable.Create<IDataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<IDataPacket> o) {
-         var adapter = new BlueTableImmutableAdapter<T>(schema);
-         IDataPacket initDataPacket = new DataPacket(schema);
-         return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
-
-         (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
-            (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
-            var bd = state.Adapter.ToDataPacketBuilder();
-            bd.Update(tableUpdate);
-            var newRows = bd.ToImmutable(out var dp);
-            return (newRows, dp);
-         }
-      }
-   }
-
-   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.AsDataPackets(schema).AsViewStream();
-
-   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
-      var indices = ResolveIndices(schema, new[] { name });
-      return indices[0];
-   }
-
-   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
-      var indices = ResolveIndices(schema, new[] { name1, name2 });
-      return (indices[0], indices[1]);
-   }
-
-   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
-      return (indices[0], indices[1], indices[2]);
-   }
-
-   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
-      return (indices[0], indices[1], indices[2], indices[3]);
-   }
-
-   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
-      var columns = schema.columnInfo;
-      var indices = new int[columnNames.Length];
-      int nResolved = 0;
-      for (int i = 0; i < columnNames.Length; i++) {
-         int origNResolved = nResolved;
-         for (int j = 0; j < columns.Length; j++) {
-            if (columns[j].colName == columnNames[i]) {
-               indices[i] = j;
-               nResolved++;
-               break;
-            }
-         }
-         if (nResolved == origNResolved) {
-            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
-         }
-      }
-      return indices;
-   }
-
-   public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
-      Observable.Create<IDataPacket>(o => {
-         var inputGate = new AtomicGate();
-         var outputGate = new AtomicGate();
-         BlueTableImmutable? lastTable = null;
-         BlueTableImmutable.DataPacketBuilder? bd = null;
-         IDataPacket? pendingDpOut = null;
-         return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
-
-         void OnNext(IDataPacket value) {
-            using (inputGate.BeginTransaction()) {
-               lastTable ??= new(value.newRows.schema);
-               bd ??= lastTable.ToDataPacketBuilder();
-               if (pendingDpOut is null) {
-                  // The last outgoing datapacket was just fired. Don't merge this update with the
-                  // previous one (otherwise its adds/removes would be fired twice).
-                  bd.Reset();
-               }
-               foreach (IBlueRow r in value.deletedRows) {
-                  bd.Remove(r.Id());
-               }
-               foreach (IBlueRow r in value.addedRows) {
-                  bd.AddOrUpdate(r);
-               }
-               pendingDpOut = bd.ToDataPacket(out lastTable);
-            }
-         }
-
-         void DoWork() {
-            IDataPacket? dpOut = null;
-            if (outputGate.TryEnter()) {
-               using var exit = Disposable.Create(outputGate.Exit);
-               (dpOut, pendingDpOut) = (pendingDpOut, dpOut);
-               if (dpOut is not null) {
-                  o.OnNext(dpOut);
-               }
-            }
-         }
-      });
-
-
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -1,9 +1,8 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using System;
    using BCG;
    using System.Collections.Generic;
    using Bts.DataStreamExpressions;
-   using System.Xml;
 
    // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
    // So for example, if an expresison is passed 
@@ -12,10 +11,9 @@ namespace bts.BlueOrb {
       private readonly string _dsl;
       public DSLStream(string dsl) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
-         // Remove all colons
-         string sanitizedDsl = dsl.Replace(":", "COLON");
          _dsl = dsl;
-         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         ViewStream rs = build_stream(dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
          mySetRealSource(rs);
       }
 
@@ -26,10 +24,9 @@ namespace bts.BlueOrb {
             return rv;
 
          } catch (Exception ex) {
-            //send to blueorb
             Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,13 +1,14 @@
 namespace Bts.DataStream;
 
 using BCG;
+using BCG.Extensions;
 using BCG.Utilities.General;
 
 using bts;
+using bts.collections;
 
 using Bts.DataStreamCore;
 
-using Spark.Core.Collections;
 using Spark.Core.Disposables;
 
 using System;

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -4,21 +4,14 @@ using BCG;
 
 using bts;
 
-using Microsoft.Extensions.Logging;
-
 using System;
 
-public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDisposable {
-   readonly IObservable<IDataPacket> source;
-   readonly string? name;
-   readonly ILogger? logger;
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<IDataPacket> source, string? name = null, ILogger? logger = null) {
+   public ObservableViewStream(IObservable<DataPacket> source) =>
       this.source = source;
-      this.name = name;
-      this.logger = logger;
-   }
 
    public void Dispose() {
       GC.SuppressFinalize(this);
@@ -30,13 +23,7 @@ public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDispo
 
    protected override void myOnFirstSubscriber() {
       Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
-
-      void OnError(Exception exception) =>
-         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
-
-      void OnCompleted() =>
-         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
+      subscription = source.Subscribe(myFireEvent);
    }
 
    protected override void myOnLastSubscriber() {
@@ -45,6 +32,6 @@ public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDispo
       subscription = null;
    }
 
-   IDisposable IObservable<IDataPacket>.Subscribe(IObserver<IDataPacket> observer) =>
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
       source.Subscribe(observer);
 }

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,25 +1,14 @@
-using bts;
-
-using Bts.DataStream;
-using Bts.DataStreamCore;
-using Bts.DataStreamExpressions;
-using Bts.PubSub;
-
-using Microsoft.Extensions.Hosting;
-
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Reactive.Linq;
-using System.Threading;
-using System.Threading.Tasks;
+
+using Bts.DataStreamExpressions;
+using BCG.Utilities.General;
+using bts;
 
 namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IHostedService {
+   public class RemoteQueryManager : IWorker {
       public class StreamPubUnit {
          public string name;
          public string stream_expression;
@@ -50,45 +39,23 @@ namespace BCG {
       }
       private IDictionary<string, StreamPubUnit>
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      readonly IPubTableBroker _broker;
-      readonly IManagedSubscriber _sub;
-      readonly object syncRoot = new();
-      IDisposable? cloudStreamsSub;
-      IDisposable? remoteRequestsSub;
-      bool isDisposed;
+      private ViewStream _stream = new TableStream("cloud_streams");
+      private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
-         _sub = managedSubscriber;
-         _broker = pubTableBroker;
+      public RemoteQueryManager() {
+         _stream.onChange += _on_str;
+         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
       }
-
-      public Task StartAsync(CancellationToken cancellationToken) {
-         lock (syncRoot) {
-            Log.assert(!isDisposed);
-            Log.isnull(cloudStreamsSub);
-            Log.isnull(remoteRequestsSub);
-            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
-            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
-               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-            });
+      private void _on_str(object obj)
+      {
+         if (obj is not DataPacket dp) {
+            return;
          }
-         return Task.CompletedTask;
-      }
-
-      public Task StopAsync(CancellationToken cancellationToken) {
-         lock (syncRoot) {
-            if (!isDisposed) {
-               isDisposed = true;
-               Disposable.dispose(ref cloudStreamsSub);
-               Disposable.dispose(ref remoteRequestsSub);
-            }
-         }
-         return Task.CompletedTask;
-      }
-
-      void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
@@ -105,10 +72,7 @@ namespace BCG {
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
          if (persist) {
-            string tempDir = Path.GetTempPath();
-        Directory.CreateDirectory(tempDir); // make sure it exists
-
-        stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
+            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
          StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
@@ -118,5 +82,9 @@ namespace BCG {
       private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
+
+      public void doWork() {
+
+      }
    }
 }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -9,7 +9,7 @@ namespace BCG {
    [DataStreamUserRespresentation("remoteQuery")]
    public class RemoteQueryStream : ViewStream {
       private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = PublisherCore.Create();
+      private PublisherCore _publisher_core = new PublisherCore();
       private ViewStream _source_str;
 
       public RemoteQueryStream(params string[] lines) {

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -2,6 +2,9 @@ namespace bts {
    using System.IO;
    using BCG;
 
+   using System.IO;
+   using System.Threading.Tasks.Dataflow;
+
    public class RemoteRequest {
       public readonly string stream_expression;
       public readonly string published_table_name;
@@ -25,4 +28,4 @@ namespace bts {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,46 +1,33 @@
-using Spark.Core.Collections;
-
 using System;
-using System.Collections.Generic;
-using System.Linq;
 
-namespace BCG {   
+namespace BCG {
    public class AddColumnsStream : ViewStream {
-      public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
          string[] toks = exp.Split(':');
-         string col_name = toks[0].Trim();
-         string col_value = toks[1].Trim();
+         string col_name = toks[0];
+         string col_value = toks[1];
          Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return (col_name, tup.Item1, tup.Item2);
+         return Tuple.Create(col_name, tup.Item1, tup.Item2);
       }
 
-      public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
-         exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
-
-      readonly List<(string Name, Type Type, object Value)> _new_col_map;
-      IBlueTableSchema? _out_schema;
+      private Tuple<string, Type, object> _new_col;
+      private IBlueTableSchema _out_schema;
 
       public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col_map = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, _get_schema, _map);
+         _new_col = parse_exp(col_def);
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
          mySetRealSource(rs);
       }
 
       private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null) {
-            _out_schema = input_schema;
-            foreach (var m in _new_col_map) {
-               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
-            }
-         }
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
          return _out_schema;
       }
 
       private IBlueRow _map(IBlueRow r) {
          MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         foreach (var m in _new_col_map) {
-            rv.setValue(m.Name, m.Value);
-         }
+         rv.setValue(_new_col.Item1, _new_col.Item3);
          return rv;
       }
    }

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs

@@ -1,10 +1,9 @@
-using mbts;
-
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
 
 namespace BCG
 {
@@ -34,7 +33,7 @@ namespace BCG
 
          private void _stream_onChange(object newValue)
          {
-            var dp = (IDataPacket)newValue;
+            DataPacket dp = newValue as DataPacket;
             if (dp != null && dp.newRows.Count > 0)
             {
                _stream.onChange -= _stream_onChange;
@@ -43,8 +42,8 @@ namespace BCG
          }
       }
 
-      public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
-      public delegate string GetColumnPrefix(object?[] transposing_col_values);
+      public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
+      public delegate string GetColumnPrefix(object[] transposing_col_values);
 
       public delegate string GetColumnSuffix(string col_name);
       private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
@@ -52,31 +51,33 @@ namespace BCG
       private IncrementValues _increment_values;
       private GetColumnPrefix _get_column_prefix;
 
-      IBlueTableSchema? _schema;
+      private IBlueTableSchema _schema;
 
       private string[] _group_by_cols;
       private Pair<string, Type>[] _cols_per_transposed;
       private string[] _transposing_cols;
 
-      int[]? _transposing_col_indices;
+      private int[] _transposing_col_indices;
 
-      Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
+      private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
          : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
                 get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
       { }
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
       {
          if (get_column_suffix_del != null)
             _get_column_suffix = get_column_suffix_del;
          if (transposed_col_value_comparer != null)
          {
-            _transpsed_col_value_comparer_wrapper = new((p1, p2) =>
-               transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
+            _transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
+               {
+                  return transposed_col_value_comparer(p1.second, p2.second);
+               });
          }
          _group_by_cols = group_by_cols.Split(' ');
          _transposing_cols = transposed_cols_group.Split(' ');
@@ -105,7 +106,7 @@ namespace BCG
             col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
          foreach (string tc_prefix in tc_prefixes)
             foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
          IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
          if (_schema == null)
             _schema = schema;
@@ -129,14 +130,19 @@ namespace BCG
 
       public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
       {
-         var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
-         var group_by_cols_value_dict = new Dictionary<int, object?[]>();
+         Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
+         Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
          BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
          Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
          string table_name = string.Empty;
          if (grouped_rows.Count > 0)
          {
-            var schema = grouped_rows.Values.First()[0].schema;
+            IBlueTableSchema schema = null;
+            foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
+            {
+               schema = kvp.Value[0].schema;
+               break;
+            }
             table_name = schema.tableName;
             _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
             int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
@@ -149,22 +155,23 @@ namespace BCG
          else
             return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); 
          Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
-         var tc_values = new List<Pair<int, object?[]>>();
+         List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
          foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
          {
             //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values 
-            var incrementing_values = new Dictionary<int, object?[]>();
+            Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
             foreach (IBlueRow r in kvp.Value)
             {
                int hash = hasher.get_hash(r, _transposing_col_indices);
-               if (!incrementing_values.TryGetValue(hash, out var values))
+               object[] values;
+               if (!incrementing_values.TryGetValue(hash, out values))
                {
                   incrementing_values.Add(hash, _get_per_transposing_col_values(r));
                   string prefix = _get_column_prefix(_get_transposing_col_values(r));
                   if (!tc_prefixes.ContainsKey(hash))
                   {
                      tc_prefixes.Add(hash, prefix);
-                     tc_values.Add(new(hash, _get_transposing_col_values(r)));
+                     tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
                   }
                }
                else
@@ -187,18 +194,18 @@ namespace BCG
          _set_schema(prefixes, group_by_col_info, table_name);
          BlueTable result = new BlueTable(_schema);
          int id = 1;
-         foreach (var kvp in all_tc_infos_and_values)
+         foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
          {
             MutableBlueRow mbr = new MutableBlueRow(_schema);
             mbr.setValue("id", id++);
             for (int i = 0; i < _group_by_cols.Length; i++)
                mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
-            foreach (var tc_cols in kvp.Value)
+            foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
             {
                string prefix = tc_prefixes[tc_cols.Key];
                for (int i = 0; i < _cols_per_transposed.Length; i++)
                {
-                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
+                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
                   if(mbr.schema.columnNames.Contains(column))
                      mbr.setValue(column, tc_cols.Value[i]);
                }
@@ -210,25 +217,28 @@ namespace BCG
 
       #endregion
 
-      object?[] _get_transposing_col_values(IBlueRow row) {
-         var values = new object?[_transposing_cols.Length];
+      private object[] _get_transposing_col_values(IBlueRow row)
+      {
+         object[] values = new object[_transposing_cols.Length];
          for (int i = 0; i < _transposing_cols.Length; i++)
             values[i] = row[_transposing_cols[i]];
          return values;
       }
 
-      object?[] _get_per_transposing_col_values(IBlueRow row) {
-         var values = new object?[_cols_per_transposed.Length];
+      private object[] _get_per_transposing_col_values(IBlueRow row)
+      {
+         object[] values = new object[_cols_per_transposed.Length];
          for (int i = 0; i < _cols_per_transposed.Length; i++)
-            values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
+            values[i] = row[_cols_per_transposed[i].first];
          return values;
       }
 
-      object?[] _get_group_by_cols_values(IBlueRow row) {
-         var values = new object?[_group_by_cols.Length];
+      private object[] _get_group_by_cols_values(IBlueRow row)
+      {
+         object[] values = new object[_group_by_cols.Length];
          for (int i = 0; i < values.Length; i++)
             values[i] = row[_group_by_cols[i]];
          return values;
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -1,47 +1,52 @@
 using System;
 
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
       private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
       private int myId = 0;
       private ViewStream myVS;
 
-      public CountStream(ViewStream vs) {
+      public CountStream (ViewStream vs)
+      {
          myVS = vs;
       }
 
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber () {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber () {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
 
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
             myLastFiredDataPacket =
                myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
                   :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
                   myLastFiredDataPacket.newRows)
             );
       }
 
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
          retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -1,19 +1,22 @@
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
 namespace BCG
 {
    public class DerivativeStream : ViewStream 
    {
-      IBlueTableSchema? mySchema;
+
+      private IBlueTableSchema mySchema;
       private string myDerivandColumn;
       private string myOutputColumn;
       private string myKeyColumn;
       private DateTime myLastPacketArrival = DateTime.MinValue;      
       private ViewStream myDataSource;
-      Dictionary<string, double>? myPreviousValues;
+      
+      private IDictionary<string, double> myPreviousValues;
+
 
       public DerivativeStream(ViewStream input, string keyColumn,  string columnToDifferentiate, string outputColName)
       {
@@ -36,9 +39,9 @@ namespace BCG
 
       void myDataSource_onChange(object newValue)
       {
-         var dp = (IDataPacket)newValue;
+         DataPacket dp = (DataPacket)newValue;
          if (myLastPacketArrival == DateTime.MinValue)
-            mySchema = myBuildOutputSchema(dp.Schema);
+            mySchema = myBuildOutputSchema(dp.schema);
          else
          {
             DateTime now = DateTime.Now;
@@ -52,7 +55,7 @@ namespace BCG
             else
             {
                IBlueRowCollection deleted, added;
-               BlueDataUtility.computeDifferences(thisVersion, ((IDataPacket)myLastValue).newRows, out deleted, out added);
+               BlueDataUtility.computeDifferences(thisVersion, ((DataPacket)myLastValue).newRows, out deleted, out added);
                myFireEvent(new DataPacket(thisVersion, added, deleted));  
             }
          }
@@ -61,13 +64,12 @@ namespace BCG
 
       protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
       {
-         ExceptionAssert.NotNull(mySchema, myPreviousValues);
          object[] outValues = new object[mySchema.columnInfo.Length];
          r.Values.CopyTo(outValues, 0);
 
-         var derivand = r[myDerivandColumn];
-         var key = getRowKey(r);
-         if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
+         object derivand = r[myDerivandColumn];
+         double lastValue = default(double);
+         if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value)
             outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
          else
             outValues[mySchema.columnInfo.Length - 1] = 0.0;
@@ -82,19 +84,16 @@ namespace BCG
          cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double));
          return new BlueTableSchema("derviative of " + inSchema.tableName, cols);
       }
-
-      protected void myStoreNewPacketInfo(IDataPacket dp) {
+      protected void myStoreNewPacketInfo(DataPacket dp)
+      {
          myPreviousValues = new Dictionary<string, double>();
          myLastPacketArrival = DateTime.Now;
          foreach (IBlueRow row in dp.newRows)
          {
-            var derivand = row[myDerivandColumn];
+            object derivand = row[myDerivandColumn];
             if (derivand != null && derivand != DBNull.Value)
-               myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
+               myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand);
          }
       }
-
-      string getRowKey(IBlueRow row) =>
-         row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
    }
 }

src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs

@@ -1,9 +1,7 @@
-using mbts;
-
-using Spark.Core.Collections;
-
-using System;
+using System;
+using System.Collections;
 using System.Collections.Generic;
+using mbts;
 
 namespace BCG
 {
@@ -14,9 +12,9 @@ namespace BCG
    {
       ViewStream myLStream, myRStream;
 
-      DataPacket? myLastR;
-      DataPacket? myLastL;
-      BlueTable? myLastFired;
+      DataPacket myLastR = null, myLastL = null;
+
+      BlueTable myLastFired = null;
 
       double myThreshold = .00005;
 
@@ -26,14 +24,14 @@ namespace BCG
       public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : 
          this(null, v1, v2, thresh, ColumnsToMatch) { }
 
-      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
       {
          myThreshold = thresh;
          myLStream = v1;
          myRStream = v2;
          myColumnsToMatch = columnsToMatch;
       }
-      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
          : this(db,v1,v2,.00005,columnsToMatch)
       {
       }
@@ -52,22 +50,45 @@ namespace BCG
          myRStream.onChange -= myRChange;
       }
 
+      struct ListComparer : IEqualityComparer<List<object>>
+      {
+         public bool Equals(List<object> x, List<object> y)
+         {
+            if (x.Count != y.Count)
+               return false;
+
+            for (int i = 0; i < x.Count; ++i)
+               if (!x[i].Equals(y[i]))
+                  return false;
+            return true;
+         }
+
+         public int GetHashCode(List<object> obj)
+         {
+            int ret = 0;
+            for(int i = 0; i < obj.Count; ++i)
+               ret ^= obj[i].GetHashCode();
+            return ret;
+         }
+      }
+
       void myHandleCallbacks()
       {
-         DataPacket? packet;
+         DataPacket packet = null;
+
          if (myLastL == null || myLastR == null)
          {
             packet = myLastL == null ? myLastR : myLastL;
-            if (packet is not null) {
+            if (packet != null)
                myFireEvent(packet);
-               myLastFired = new BlueTable(packet.newRows);
-            }
+
+            myLastFired = new BlueTable(packet.newRows);
             return;
          }
 
          BlueTable table = new BlueTable(myLastL.newRows.schema);
 
-         var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
+         Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
 
          foreach(BlueRow row in myLastL.newRows)
          {
@@ -97,7 +118,11 @@ namespace BCG
                }
                catch (Exception) { }
             }
-            rowPairs.TryGetValue(hash, out var pair);
+
+            Pair<BlueRow, BlueRow> pair;
+            rowPairs.TryGetValue(hash, out pair);
+
+
             if (pair == null)
                pair = new Pair<BlueRow, BlueRow>(null, row);
             else
@@ -113,7 +138,8 @@ namespace BCG
 
             if (pair.first == null || pair.second == null)
             {
-               var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
+               BlueRow row = pair.first == null ? pair.second : pair.first;
+
                Dictionary<string, object> newRow = new Dictionary<string, object>();
                foreach (ColumnInfo info in row.schema.columnInfo)
                {
@@ -185,4 +211,4 @@ namespace BCG
 
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs

@@ -14,8 +14,10 @@ namespace BCG
 		public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
          : base (colMapExpression, null, columnNames) {}
 
-      public override IBlueRow? evaluate(IDataPacket packet) {
-         myOutputTable ??= new GroupByOutputTable(packet.Schema);
+
+      public override IBlueRow evaluate (DataPacket packet)
+      {
+         myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
          string hashString = GroupByStream.getRepString (packet, myGroupByCols);
          if (packet.newRows.Count == 0)
             //if there are no rows for this group we remove the group from the output table

src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs

@@ -5,7 +5,7 @@ namespace BCG
 {
 	public class FileViewStream : ViewStream
 	{
-      DataPacket? myDp;
+      private DataPacket myDp;
 		public FileViewStream (string dataFileName, 
          string schemaFileName)
          : this (dataFileName, schemaFileName, Int32.MaxValue)

src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs

@@ -1,7 +1,8 @@
-using BCG.Utilities.General;
-
-using System;
-
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 namespace BCG
 {
     public class DateBasedFilter: IDynamicStreamFilter, IWorker
@@ -25,14 +26,20 @@ namespace BCG
             myReference = myTodaysDate.AddDays(-1 * myNDays);                     
         }       
         
-        public event FilterChangeDelegate? onFilterChange;       
+        public event FilterChangeDelegate onFilterChange;       
         
-        public bool filter(IBlueRow r) =>
-           r[myDateColumn] switch {
-              DBNull or null => true,
-              DateTime t => t >= myReference,
-              _ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
-           };
+        public bool filter(IBlueRow r)
+        {
+            if (DBNull.Value.Equals(r[myDateColumn]))
+            {
+                //return false;
+                return true; //it can be considered as our assumption  whenever datecolumn is db null we return true 
+            }
+            DateTime t = (DateTime)r[myDateColumn];                   
+            bool returnVallue =  t < myReference ? false : true;
+            return returnVallue;
+        }
+              
 
         public void doWork()
         {
@@ -40,7 +47,7 @@ namespace BCG
             {
                 myTodaysDate = DateTime.Today;
                 myReference = myTodaysDate.AddDays(-1 * myNDays);
-                onFilterChange?.Invoke();
+                onFilterChange();
             }         
         }      
     }

src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs

@@ -1,6 +1,8 @@
-using BCG.Utilities.General;
-
-using System;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 
 namespace BCG
 {
@@ -24,7 +26,7 @@ namespace BCG
 
       #region IDynamicStreamFilter Members
 
-      public event FilterChangeDelegate? onFilterChange;
+      public event FilterChangeDelegate onFilterChange;
 
       #endregion
 
@@ -32,10 +34,7 @@ namespace BCG
 
       public bool filter(IBlueRow r)
       {
-         DateTime rowDate = r[myDateColName] switch {
-            DBNull or null => DateTime.Now.Date,
-            object obj => (DateTime)obj,
-         };
+         DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date;
          DateTime today = DateTime.Now.Date;
          return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
       }
@@ -49,7 +48,7 @@ namespace BCG
          if (DateTime.Now.Date != myLastCheckedTime.Date)
          {
             myLastCheckedTime = DateTime.Now;
-            onFilterChange?.Invoke();
+            onFilterChange();
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,109 +1,93 @@
-using bts;
+using Bts.DataStream;
 
-using Bts.DataStream.Primitives.Filter;
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-using Spark.Core.Disposables;
+using mbts.config;
 
 using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
 using System.Text.RegularExpressions;
 
-namespace BCG.StreamWorld.Addition {
-   [DataStreamUserRespresentation(false)]
-   public class FilterStream : ViewStream {
+namespace BCG.StreamWorld.Addition
+{
+   [DataStreamUserRespresentation (false)]
+   public class FilterStream : ViewStream
+   {
       private ViewStream myBaseStream;
-      BlueTable? myBaseTable;
-      BlueTable? myStateTable;
+      private BlueTable myBaseTable, myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
-         : this(vs, "1=1") { }
+         : this(vs, "1=1")
+      { }
 
       public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp)) { }
+         : this(vs, new BooleanExpressionFilter(filterExp))
+      { }
 
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
+      public FilterStream(ViewStream vs, IStreamFilter filter)
+      {
          myBaseStream = vs;
          myFilter = filter;
-         vs.ErrorChanged += OnError;
-         if (filter is IDynamicStreamFilter dfilter)
-            dfilter.onFilterChange += myHandleFilterChange;
+         if (filter is IDynamicStreamFilter)
+            (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
       }
 
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
-         myBaseTable?.clear();
-         myStateTable?.clear();
+         myBaseTable.clear();
+         myStateTable.clear();
       }
 
-      bool filterRow(IBlueRow r) {
-         try {
-            return myFilter.filter(r);
-         } catch (Exception ex) {
-            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+      private void myHandleChanges(object newValue)
+      {
+         DataPacket dp = newValue as DataPacket;
+         if (myStateTable == null)
+         {
+            bts.Log.assert(myBaseTable == null);
+            myStateTable = new BlueTable(dp.schema);
+            myBaseTable = new BlueTable(dp.schema);
          }
-         return false;
-      }
-
-      private void myHandleChanges(object newValue) {
-         try {
-            var dp = (IDataPacket)newValue;
-            var schema = dp.Schema;
-            init(schema);
-            var dels = new BlueTable(schema);
-            var adds = new BlueTable(schema);
-            foreach (IBlueRow r in dp.deletedRows) {
-               myBaseTable.deleteRow(r);
-               if (myStateTable.containsID(r.Id())) {
-                  dels.addRow(r);
-                  myStateTable.deleteRow(r);
-               }
+         BlueTable dels = new BlueTable(dp.schema),
+                     adds = new BlueTable(dp.schema);
+         foreach (IBlueRow r in dp.deletedRows)
+         {
+            myBaseTable.deleteRow(r);
+            if (myStateTable.containsID((int)r["id"]))
+            {
+               dels.addRow(r);
+               myStateTable.deleteRow(r);
             }
-            foreach (IBlueRow r in dp.addedRows) {
-               myBaseTable.addRow(r);
-               if (filterRow(r)) {
-                  adds.addRow(r);
-                  myStateTable.addRow(r);
-               }
-            }
-            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
-               myFireEvent(new DataPacket(myStateTable, adds, dels));
-         } catch (Exception ex) {
-            OnError(ex);
          }
+         foreach (IBlueRow r in dp.addedRows)
+         {
+            myBaseTable.addRow(r);
+            if (myFilter.filter(r))
+            {
+               adds.addRow(r);
+               myStateTable.addRow(r);
+            }
+         }
+         if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
+            myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
-      public IStreamFilter filter {
-         set {
+      public IStreamFilter filter
+      {
+         set
+         {
             myFilter = value;
-            if (myBaseTable != null) {
-               ExceptionAssert.NotNull(myStateTable);
+            if (myBaseTable != null)
+            {
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
-               // If the filter throws an exception, we can rethrow it as a StreamFilterException
-               // and include the 'currentRow' (for which the filter failed) to provide context.
-               IBlueRow? currentRow = null;
-               foreach (IBlueRow r in myBaseTable) {
-                  try {
-                     currentRow = r;
-                     if (filterRow(r)) {
-                        myStateTable.addRow(r);
-                     }
-                  } catch (Exception ex) {
-                     Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-                     OnError(new StreamFilterException(StreamId, StreamName, null, ex, currentRow));
-                  }
-               }
+               foreach (IBlueRow r in myBaseTable)
+                  if (myFilter.filter(r))
+                     myStateTable.addRow(r);
                IBlueRowCollection dels, adds;
                BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
                myFireEvent(new DataPacket(myStateTable, adds, dels));
@@ -115,17 +99,6 @@ namespace BCG.StreamWorld.Addition {
       //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
 
       private void myHandleFilterChange() { filter = myFilter; }
-
-      [MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
-      void init(IBlueTableSchema schema) {
-         if (myStateTable is null) {
-            Log.isnull(myBaseTable);
-            myStateTable = new BlueTable(schema);
-            myBaseTable = new BlueTable(schema);
-         } else {
-            Log.notnull(myBaseTable);
-         }
-      }
    }
 }
 
@@ -149,24 +122,29 @@ namespace BCG
       }
 
       #region IStreamFilter Members
-      public bool filter(IBlueRow dr) =>
-         myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
-            DBNull or null => string.Empty,
-            object obj => obj.ToString() ?? string.Empty
-         }));
+      public bool filter(IBlueRow dr)
+      {
+         bool rv = true;
+         for (int i = 0; i < myCols.Length; i++)
+         {
+            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+            if (!rv)
+               return rv;
+         }
+         return rv;
+      }
       #endregion
    }
 
    public class FilterStream : ViewStream
    {
-      BlueTable? myLastFiredRows;
-      IBlueRowCollection? myLastReceivedRows;
-      IStreamFilter? myFilter;
-      readonly ViewStream? myParentStream;
-      string? myFilterString;
-      readonly StreamWorld.Addition.FilterStream? myFilterStream;
-      readonly bool myUseNewFilter = false;
-      IDisposable? subscription;
+      private BlueTable myLastFiredRows;
+      private IBlueRowCollection myLastReceivedRows;
+      private IStreamFilter myFilter;
+      private ViewStream myParentStream;
+      private string myFilterString;
+      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
+      private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -196,10 +174,9 @@ namespace BCG
          {
             myParentStream = vs;
             myFilter = filter;
-            myFilterString = filter.ToString() ?? string.Empty;
-            if (filter is IDynamicStreamFilter dfilter) {
-               dfilter.onFilterChange += myHandleFilterChange;
-            }
+            myFilterString = filter.ToString();
+            if (filter is IDynamicStreamFilter)
+               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
          }
       }
 
@@ -223,20 +200,18 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber() {
-         ExceptionAssert.IsNull(subscription);
-         subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
-      }
+      protected override void myOnFirstSubscriber()
+      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         Disposable.Dispose(ref subscription);
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
       }
 
       protected override void myFireEvent(object o)
       {
-         var dp = (IDataPacket)o;
+         DataPacket dp = o as DataPacket;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -245,39 +220,34 @@ namespace BCG
       private void myHandleCallbacks(object newValue)
       {
          DataPacket retVal;
-         var arrivedPacket = (IDataPacket)newValue;
+         DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
-         try {
-            if (myLastFiredRows != null)
-            {
-               var addedRows = new BlueTable(myLastFiredRows.schema);
-               var deletedRows = new BlueTable(myLastFiredRows.schema);
-               retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
-               foreach (IBlueRow r in arrivedPacket.deletedRows)
-                  if (myLastFiredRows.containsID(r.Id()))
-                  {
-                     myLastFiredRows.deleteRow(r);
-                     deletedRows.addRow(r);
-                  }
-               foreach (IBlueRow r in arrivedPacket.addedRows)
-                  if (filterRow(r))
-                  {
-                     myLastFiredRows.addRow(r);
-                     addedRows.addRow(r);
-                  }
-            }
-            else
-            {
-               myLastFiredRows = new BlueTable(arrivedPacket.Schema);
-               retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-               foreach (IBlueRow r in arrivedPacket.newRows)
-                  if (filterRow(r))
-                     myLastFiredRows.addRow(r);
-            }
-            myFireEvent(retVal);
-         } catch (Exception ex) {
-            OnError(ex);
+         if (myLastFiredRows != null)
+         {
+            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+               new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.deletedRows)
+               if (myLastFiredRows.containsID((int)r["id"]))
+               {
+                  myLastFiredRows.deleteRow(r);
+                  (retVal.deletedRows as BlueTable).addRow(r);
+               }
+            foreach (IBlueRow r in arrivedPacket.addedRows)
+               if (myFilter.filter(r))
+               {
+                  myLastFiredRows.addRow(r);
+                  (retVal.addedRows as BlueTable).addRow(r);
+               }
          }
+         else
+         {
+            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.newRows)
+               if (myFilter.filter(r))
+                  myLastFiredRows.addRow(r);
+         }
+         myFireEvent(retVal);
       }
 
       public IStreamFilter filter
@@ -286,7 +256,7 @@ namespace BCG
          {
             if (myUseNewFilter)
             {
-               requireNewFilter().filter = filter;
+               myFilterStream.filter = filter;
             }
             else
             {
@@ -295,29 +265,25 @@ namespace BCG
                {
                    if (myLastReceivedRows != null)
                    {
-                     try {
                        BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
                        foreach (IBlueRow r in myLastReceivedRows)
-                           if (filterRow(r))
+                           if (myFilter.filter(r))
                                nRows.addRow(r);
                        IBlueRowCollection dRows = new BlueTable(nRows.schema),
                           aRows = new BlueTable(nRows.schema);                      
                        BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
                        myLastFiredRows = nRows;
                        myFireEvent(new DataPacket(nRows, aRows, dRows));
-                     } catch (Exception ex) {
-                        OnError(ex);
-                     }
-                  }
+                   }
                }
             }
          }
          get
          {
             if (myUseNewFilter)
-               return requireNewFilter().filter;
+               return myFilterStream.filter;
             else
-               return requireOldFilter();
+               return myFilter;
          }
       }
 
@@ -330,25 +296,7 @@ namespace BCG
 
       private void myHandleFilterChange()
       {
-         filter = requireOldFilter();
+         filter = myFilter;
       }
-
-      bool filterRow(IBlueRow r) {
-         try {
-            return requireOldFilter().filter(r);
-         } catch (Exception ex) {
-            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
-         }
-         return false;
-      }
-
-      [MemberNotNull(nameof(myFilterStream))]
-      StreamWorld.Addition.FilterStream requireNewFilter() =>
-         ExceptionAssert.MemberNotNull(myFilterStream);
-
-      [MemberNotNull(nameof(myFilter))]
-      IStreamFilter requireOldFilter() =>
-         ExceptionAssert.MemberNotNull(myFilter);
    }
 }

src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs

@@ -1,6 +1,8 @@
-using BCG.Utilities.General;
-
-using System;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 
 namespace BCG
 {    
@@ -27,7 +29,7 @@ namespace BCG
             myReference = myCurrentTime.AddSeconds(-1 * myNSecs);                  
         }
         
-        public event FilterChangeDelegate? onFilterChange;       
+        public event FilterChangeDelegate onFilterChange;       
         
         public bool filter(IBlueRow r)
         {
@@ -36,8 +38,9 @@ namespace BCG
                 //it can be considered as our assumption  whenever datecolumn is db null we return true 
                 return true;  //dont use this for volparamsched use true              
             }
-            var t = (DateTime)r[myDateColumn]!;
-            return t >= myReference;
+            DateTime t = (DateTime)r[myDateColumn];                   
+            bool returnVallue =  t < myReference ? false : true;
+            return returnVallue;
         }
               
 
@@ -48,7 +51,8 @@ namespace BCG
                 myCurrentTime = DateTime.Now;
                 myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
                 myLastTimeFilteredApplied = DateTime.Now;              
-                onFilterChange?.Invoke();
+                onFilterChange();
+                
             }           
         }
     }

src/DataStreamSuite/DataStream/Primitives/Filter/StreamFilterException.cs

@@ -1,19 +0,0 @@
-#nullable enable
-namespace Bts.DataStream.Primitives.Filter;
-
-using BCG;
-
-using System;
-
-public class StreamFilterException : Exception {
-   public StreamFilterException(Guid streamId, string streamName, string? message, Exception? innerException, IBlueRow? badRow = null)
-      : base(message, innerException) {
-      StreamId = streamId;
-      StreamName = streamName;
-      BadRow = badRow;
-   }
-
-   public Guid StreamId { get; }
-   public string StreamName { get; }
-   public IBlueRow? BadRow { get; }
-}

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -52,14 +52,14 @@ namespace BCG {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
          myVsd = vsd;
          _time_col = rowTimeColumnName;
          _ttl = rowTimeToLive;
-         _filter_period = filter_period;
+         myFilterFrequency = filterFrequency;
          myNextFilter = DateTime.MinValue;
 
          // add worker
@@ -71,10 +71,11 @@ namespace BCG {
 
       #region IStreamFilter
       public bool filter(IBlueRow row) {
-         DateTime now_utc = DateTime.UtcNow;
-         DateTime row_time_utc = row.get_datetime(_time_col, DateTime.Now).ToUniversalTime();
-         if (now_utc > row_time_utc) {
-            TimeSpan span = now_utc - row_time_utc;
+         DateTime now = DateTime.Now;
+
+         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
+         if (now > row_time) {
+            TimeSpan span = now - row_time;
             if (span >= _ttl) {
                return (false);
             }
@@ -89,9 +90,9 @@ namespace BCG {
 
          if (now > myNextFilter) {
             // fire event
-            onFilterChange?.Invoke();
+            onFilterChange();
 
-            myNextFilter = now.Add(_filter_period);
+            myNextFilter = now.Add(myFilterFrequency);
          }
       }
       #endregion
@@ -107,7 +108,7 @@ namespace BCG {
       private ViewStreamDatabase myVsd;
       private string _time_col;
       private TimeSpan _ttl;
-      private TimeSpan _filter_period;
+      private TimeSpan myFilterFrequency;
       private DateTime myNextFilter;
       #endregion
    }

src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs

@@ -1,13 +1,11 @@
-using BCG.Utilities.General;
-
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 
 using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
+using System.Diagnostics;
 
 namespace BCG
 {
@@ -42,12 +40,12 @@ namespace BCG
       private IComparer<IBlueRow> myRightCompare;
       protected FuzzyRowComparer myFuzzyCompare;
       protected TimeSpan myTimeWindow;
-      protected IBlueTableSchema? myLeftSchema;
-      protected IBlueTableSchema? myRightSchema;
-      protected IBlueTableSchema? myOutputSchema;
+      protected IBlueTableSchema myLeftSchema;
+      protected IBlueTableSchema myRightSchema;
+      protected IBlueTableSchema myOutputSchema;
       private int myID = 0;
 
-      protected TimeWindowTable? myStateRows;
+      protected TimeWindowTable myStateRows;
 
       private DateTime myNextCull;
       const long MinCullIntervalMS = 1000;
@@ -67,7 +65,7 @@ namespace BCG
          this.myFuzzyCompare = fuzzyCompare;
       }
 
-      public virtual IBlueTableSchema? OutputSchema
+      public virtual IBlueTableSchema OutputSchema
       {
          get
          {
@@ -152,7 +150,7 @@ namespace BCG
          int count;
          for (count = 0; count < list.Count; count++)
          {
-            if ((DateTime)list[count][timeColumn]! > expiry)
+            if ((DateTime)list[count][timeColumn] > expiry)
             {
                // Stop when we find the first row that we can keep.  Since this is sorted all others are also kept.
                break;
@@ -174,16 +172,16 @@ namespace BCG
             this.myFitness = fitness;
          }
          public IBlueRow myRightRow { get; private set; }
-         public IBlueRow? myLeftRow { get; set; }
+         public IBlueRow myLeftRow { get; set; }
          public double myFitness { get; private set; }
-         public IBlueRow? myMergedRow { get; set; }
+         public IBlueRow myMergedRow { get; set; }
 
          #region IComparable<RowMatch> Members
 
-         public int CompareTo(RowMatch? other) =>
-            other is null
-               ? 1
-               : myFitness.CompareTo(other.myFitness);
+         public int CompareTo(RowMatch other)
+         {
+            return myFitness.CompareTo(other.myFitness);
+         }
 
          #endregion
       }
@@ -264,12 +262,12 @@ namespace BCG
 
       private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
       {
-         ExceptionAssert.NotNull(myStateRows);
          // We already emitted correction.MergedRow so we need to redact that and 
          // then emit correction.left as an add since it is no longer matched
          foreach (RowMatch correction in corrections.Distinct())
          {
-            myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow));
+            Debug.Assert(correction.myMergedRow != null);
+            this.myStateRows.RemoveRow(correction.myMergedRow);
             adds.addRow(this.MergeRows(correction.myLeftRow, null));
             dels.addRow(correction.myMergedRow);
          }
@@ -287,7 +285,7 @@ namespace BCG
          int r = 0;
          for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
          {
-            if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!)
+            if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
             {
                // Advance Left
                IEnumerable<RowMatch> rm = rightMatches[l];
@@ -334,7 +332,7 @@ namespace BCG
 
       private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
       {
-         IBlueRow? merged = null;
+         IBlueRow merged = null;
          if (rightMatches != null)
          {
             foreach (RowMatch match in rightMatches)
@@ -368,9 +366,9 @@ namespace BCG
       /// Override this function to provide a custom row merge.
       /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
       /// </summary>
-      protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
+      protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
       {
-         Dictionary<string, object?> newRow = new();
+         Dictionary<string, object> newRow = new Dictionary<string, object>();
 
          if (left == null && right == null) throw new ArgumentException("oh come on.");
 
@@ -395,15 +393,14 @@ namespace BCG
             }
          }
          newRow["id"] = this.myID++;
-         newRow[NEW_TIME_COLUMN] = left is not null
-            ? left[myLeftTimeColumn]
-            : right![myRightTimeColumn];
+         newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
+
          return new BlueRow(this.OutputSchema, newRow);
       }
 
       private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
       {
-         List<RowMatch>? matches = null;
+         List<RowMatch> matches = null;
 
          // Search through all rows in list and compare with the fuzzy comparer
          for (int x = 0; x < list.Count; x++)
@@ -424,7 +421,8 @@ namespace BCG
                matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
 
                // Mark the best match for this particular right hand match
-               if (this.myBestMatch.TryGetValue(match, out var oldMatch))
+               RowMatch oldMatch;
+               if (this.myBestMatch.TryGetValue(match, out oldMatch))
                {
                   if (currentMatch.myFitness < oldMatch.myFitness)
                   {
@@ -443,7 +441,7 @@ namespace BCG
             }
          }
 
-         return matches ?? Enumerable.Empty<RowMatch>();
+         return matches;
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -42,7 +42,7 @@ namespace BCG
          return retVal;
       }
 
-      public abstract IBlueRow? evaluate(IDataPacket dp);
+      public abstract IBlueRow evaluate (DataPacket dp) ;
       #endregion
 
       #region Helper methods
@@ -69,7 +69,14 @@ namespace BCG
       protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
       #endregion
 
-      IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
-         new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project"));
+      private IBlueRow myProjectRow (IBlueRow r, string colMapExpression)
+      {
+         BlueTable auxTable = new BlueTable (r.schema);
+         auxTable.addRow (r);
+         BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable);
+         ProjectStream projStream = new ProjectStream (staticStream, colMapExpression);
+         return BlueDataUtility.extractRow (projStream.rows, 0);
+      }
+
 	}
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -5,16 +5,15 @@ using System.Collections.Generic;
 using System.Linq;
 
 namespace BCG {
-   internal class GroupByStream : MethodDrivenStream {
+   public class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
-         readonly Func<IDataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
+         private Func<DataPacket, IBlueRow> _groupby_func;
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
             _groupby_func = groupby_func;
          }
-         public IBlueRow evaluate(IDataPacket packet) =>
-            _groupby_func(packet);
+         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
 
          public string[] groupByColumns { get { return _groupby_cols; } }
 
@@ -33,7 +32,7 @@ namespace BCG {
       private IBlueTableSchema myOutputSchema;
       private int myIdGenerator = 0;
 
-      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
+      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
          : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
       public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
@@ -188,11 +187,11 @@ namespace BCG {
          }
       }
 
-      public static string getRepString(IDataPacket dp, string[] colNames) {
+      public static string getRepString(DataPacket dp, string[] colNames) {
          return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow(IDataPacket dp) {
+      public static IBlueRow getRepRow(DataPacket dp) {
          if (dp.newRows.Count > 0)
             return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)

src/DataStreamSuite/DataStream/Primitives/GroupBy/IGroupByEvaluator.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BCG
 {
    /// <summary>
@@ -12,7 +14,7 @@ namespace BCG
       /// deletetion) and the added and deletedRows being the increments. If a group has
       /// to be totally removed then a null should be returned
       /// </summary>
-      IBlueRow? evaluate(IDataPacket packet);
+      IBlueRow evaluate (DataPacket packet);
 
       /// <summary>
       /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -14,8 +14,9 @@ namespace BCG {
         public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow? evaluate(IDataPacket dp) {
-            myOutputTable ??= new GroupByOutputTable(dp.Schema);
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
             string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMaxRow = myOutputTable.getRow(hashString);
             IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,3 +1,7 @@
+using System;
+
+using BCG.Utilities;
+
 namespace BCG {
     public class MinGroupByEvaluator : MaxGroupByEvaluator {
         public MinGroupByEvaluator(string outputCol, string[] groupByCols)
@@ -6,8 +10,8 @@ namespace BCG {
         public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
            base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow? evaluate(IDataPacket dp) {
-            myOutputTable ??= new GroupByOutputTable(dp.Schema);
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
             string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMinRow = myOutputTable.getRow(stringRep);
             IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs

@@ -1,10 +1,14 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
 
 namespace BCG
 {
    public interface INonIncGroupByEvaluator
    {
-      IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-      string[]? get_group_by_cols();
+      IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+      string[] get_group_by_cols();
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,51 +1,46 @@
-using bts;
-using bts.SilverPine;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-using Spark.Core.Disposables;
-
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Text.Json;
+
+using bts;
+using bts.SilverPine;
 
 namespace BCG {
-   //[DataStreamUserRespresentation("AggregateStream")]
+   [DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]?> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>?> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
                                           Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable? rv = null;
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
-               var out_rows = _evaluate_func(g_rows);
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
                if (out_rows != null && out_rows.Count > 0) {
                   rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-                  rv.addRows(out_rows);
                }
+               rv.addRows(out_rows);
             }
             return rv;
          }
-         public string[]? get_group_by_cols() {
+         public string[] get_group_by_cols() {
             return _groupby_cols_func();
          }
 
       }
-      readonly ViewStream? _main_stream;
-      readonly INonIncGroupByEvaluator? _groupby_func;
-      BlueRowHasher? _row_hasher;
-      readonly string[]? _groupby_cols;
+      private ViewStream _main_stream;
+      private INonIncGroupByEvaluator _groupby_func;
+      private BlueRowHasher _row_hasher;
+      private string[] _groupby_cols;
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
-      IDisposable? subscription;
 
       public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -54,7 +49,7 @@ namespace BCG {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public NonIncGroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
@@ -63,7 +58,7 @@ namespace BCG {
 
       public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
-         Func<string[]?> groupby_cols_func = () => {
+         Func<string[]> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
@@ -80,17 +75,12 @@ namespace BCG {
       }
 
 
-      protected override void myOnFirstSubscriber() {
-         ExceptionAssert.IsNull(subscription);
-         ExceptionAssert.NotNull(_main_stream).Subscribe(_main_stream_onchange);
-      }
+      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
 
-      protected override void myOnLastSubscriber() =>
-         Disposable.Dispose(ref subscription);
+      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
 
       private void _main_stream_onchange(object nv) {
-         ExceptionAssert.NotNull(_groupby_func);
-         var dp = (IDataPacket)nv;
+         DataPacket dp = nv as DataPacket;
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -104,12 +94,8 @@ namespace BCG {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         var total_state = _groupby_func.evaluate(grouped_rows);
+         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
          _timer.Stop();
-         if (total_state is null) {
-            Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
-            return;
-         }
          var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;
          if (applyElapsedMs >= 50) {
@@ -118,12 +104,8 @@ namespace BCG {
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
          // TODO,
-         if (total_state != null) {
-            var deletedRows = myLastValue is null
-               ? new BlueTable(total_state.schema)
-               : ((IDataPacket)myLastValue).newRows;
-            myFireEvent(new DataPacket(total_state, total_state, deletedRows));
-         }
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs

@@ -1,6 +1,6 @@
 using System;
+using BCG.Utilities;
 using System.Collections;
-using System.Collections.Generic;
 
 namespace BCG
 {
@@ -11,10 +11,12 @@ namespace BCG
    /// </summary>
    public class RowDictionaryTable : IBlueRowCollection
    {
-      protected readonly Dictionary<string, BlueTable> myRowCollections = new();
-      protected readonly BlueTable myLinearCollection;
-      protected readonly string[] myCols;
+      protected IDictionary myRowCollections = new Hashtable ();
+      protected BlueTable myLinearCollection;
+      protected string[] myCols;
       protected IBlueTableSchema mySchema;
+      
+      protected RowDictionaryTable ()  {}
 
       public RowDictionaryTable (IBlueRowCollection rows, params string[] cols)
          : this (rows.schema, cols)
@@ -38,7 +40,7 @@ namespace BCG
 
       public virtual void addRow (IBlueRow row)
       {
-         var rows = getMutableGroup(row);
+         BlueTable rows = (BlueTable)getGroup (row);
          if (rows == null)
          {
             //i.e we dont have any group for this row => we need to create a new group
@@ -51,7 +53,7 @@ namespace BCG
 
       public virtual void removeRow (IBlueRow row)
       {
-         var rowGroup = requireMutableGroup(row);
+         BlueTable rowGroup = (BlueTable)getGroup (row);
 //         BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
          rowGroup.deleteRow (row);
          myLinearCollection.deleteRow (row);
@@ -63,7 +65,7 @@ namespace BCG
 
       public void removeGroup (IBlueRow representativeRow)
       {
-         var rowGroup = requireGroup(representativeRow);
+         IBlueRowCollection rowGroup = getGroup (representativeRow);
          // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
          foreach (IBlueRow row in rowGroup)
             myLinearCollection.deleteRow (row);
@@ -83,31 +85,24 @@ namespace BCG
       /// <returns></returns>
       //public IBlueRowCollection getGroup (IBlueRow row)  {return getGroup (
 
-      public IBlueRowCollection? getGroup (IBlueRow r) =>
-         getGroup(myRowToString(r));
+      public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));}
 
       public ICollection getGroups ()  {return myRowCollections.Values;}
 
-      public IBlueRowCollection? getGroup(string repString) =>
-         getMutableGroup(repString);
+      private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2)
+      {
+         bool retVal = true;
+         foreach (string col in myCols)
+            retVal &= r1[col].Equals(r2[col]);
+         return retVal;
+      }
 
-      BlueTable? getMutableGroup(IBlueRow row) =>
-         getMutableGroup(myRowToString(row));
-
-      BlueTable? getMutableGroup(string group) =>
-         myRowCollections.GetValueOrDefault(group);
-
-      IBlueRowCollection requireGroup(IBlueRow row) =>
-         requireGroup(myRowToString(row));
-
-      IBlueRowCollection requireGroup(string group) =>
-         getGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
-
-      BlueTable requireMutableGroup(IBlueRow row) =>
-         requireMutableGroup(myRowToString(row));
-
-      BlueTable requireMutableGroup(string group) =>
-         getMutableGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
+      public IBlueRowCollection getGroup (string repString)
+      {
+         IBlueRowCollection retVal = myRowCollections[repString] != null ?
+            (IBlueRowCollection) myRowCollections[repString] : null;
+         return retVal;
+      }
 
       private string myRowToString (IBlueRow r)  {return BlueDataUtility.rowToString (r, myCols);}
 

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BCG {
     /// <summary>
     /// Computes the sum for a given column
@@ -8,8 +10,8 @@ namespace BCG {
         public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(IDataPacket dp) {
-            myOutputTable ??= new GroupByOutputTable(dp.Schema);
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
             IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
             return sumRow;
@@ -31,7 +33,7 @@ namespace BCG {
 
         #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
 
         public string[] groupByColumns { get { return myGroupByCols; } }
 
@@ -49,8 +51,9 @@ namespace BCG {
         public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(IDataPacket dp) {
-            myOutputTable ??= new GroupByOutputTable(dp.Schema);
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+
             IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
             return sumRow;
         }
@@ -65,7 +68,7 @@ namespace BCG {
 
         #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
 
         public string[] groupByColumns { get { return myGroupByCols; } }
 

src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs

@@ -1,31 +1,31 @@
-//using System;
-//using System.Collections.Generic;
-//using System.Linq;
-//using System.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
-//namespace BCG
-//{
-//   public class FullJoinStream : ViewStream
-//   {
-//      private ViewStream myMainStream;
-//      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
-//         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
-//      {
-//         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
-//         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
-//         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
-//         myMainStream = new AppendStream(myMainStream, filtered);
-//      }
+namespace BCG
+{
+   public class FullJoinStream : ViewStream
+   {
+      private ViewStream myMainStream;
+      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
+         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
+      {
+         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
+         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
+         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
+         myMainStream = new AppendStream(myMainStream, filtered);
+      }
 
-//      protected override void myOnFirstSubscriber()
-//      {
-//         myMainStream.onChange += myHandleMainStreamChange;
-//      }
+      protected override void myOnFirstSubscriber()
+      {
+         myMainStream.onChange += myHandleMainStreamChange;
+      }
 
-//      private void myHandleMainStreamChange(object newData)
-//      {
-//         IBlueRowCollection result = (newData as DataPacket).newRows;
-//         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
-//      }
-//   }
-//}
+      private void myHandleMainStreamChange(object newData)
+      {
+         IBlueRowCollection result = (newData as DataPacket).newRows;
+         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BCG
 {
    [DataStreamUserRespresentation (false)]
@@ -15,7 +17,6 @@ namespace BCG
 
       protected override void myInitFiredPacket ()
       {
-         requireSchema();
          myLastFiredPacket = new DataPacket ();
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -24,10 +25,6 @@ namespace BCG
 
       public override void onTableClear (bool isLeft)
       {
-         if (myLastFiredPacket is null) {
-            return;
-         }
-         requireSchema();
          //incase of natural join, left or right wont matter, the end result zeroes out
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs

@@ -1,7 +1,5 @@
-using Spark.Core.Diagnostics;
-
 using System;
-using System.Diagnostics.CodeAnalysis;
+using System.Collections;
 
 namespace BCG
 {
@@ -15,10 +13,12 @@ namespace BCG
       protected JoinOperand myLeftOperand, myRightOperand;
       protected int myLeftJoinColOrdinal, myRightJoinColOrdinal;
       protected ViewStream myLeftStream, myRightStream;
-      protected JoinTableSchema? mySchema;
-      protected DataPacket? myLastFiredPacket;
+      protected JoinTableSchema mySchema;
+      protected DataPacket myLastFiredPacket;
       protected bool myOnceFired = false;
 
+      public IBlueTableSchema schema {get{return mySchema;}}
+
       public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight)
       {
          myLeftStream = vsLeft;
@@ -27,9 +27,6 @@ namespace BCG
          myRightOperand = new JoinOperand (jcRight, false);
       }
 
-      public IBlueTableSchema schema =>
-         requireSchema();
-
       protected override void myOnFirstSubscriber() 
       {
          myLeftOperand.master = this;
@@ -65,32 +62,29 @@ namespace BCG
          {
             myLeftJoinColOrdinal = myLeftOperand.joinColOridinal;
             myRightJoinColOrdinal = myRightOperand.joinColOridinal;
-            myLeftOperand.RequireInitialized();
-            myRightOperand.RequireInitialized();
             mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol, 
                myRightOperand.operandSchema, myRightOperand.joinCol);
             myInitFiredPacket ();
          }
       }
 
-      [MemberNotNull(nameof(mySchema), nameof(myLastFiredPacket))]
       protected virtual void myInitFiredPacket ()
       {
-         var schema = requireSchema();
          myLastFiredPacket = new DataPacket ();
-         myLastFiredPacket.newRows = new JoinTable (schema);
-         myLastFiredPacket.addedRows = new BlueTable (schema);
-         myLastFiredPacket.deletedRows = new BlueTable (schema);
+         myLastFiredPacket.newRows = new JoinTable (mySchema);
+         myLastFiredPacket.addedRows = new BlueTable (mySchema);
+         myLastFiredPacket.deletedRows = new BlueTable (mySchema);
       }
 
-      public IBlueTableSchema? getSchema (bool isLeft) =>
-         isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
+      public IBlueTableSchema getSchema (bool isLeft)
+      {
+         IBlueTableSchema retVal;
+         retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
+         return retVal;
+      }
 
       public void endSession ()
       {
-         if (myLastFiredPacket is null) {
-            return;
-         }
          if (!myOnceFired)
          {
             myOnceFired = true;
@@ -103,17 +97,13 @@ namespace BCG
 
       protected virtual void myInit ()
       {
-         myLeftOperand.Reset ();
-         myRightOperand.Reset ();
+         myLeftOperand.init ();
+         myRightOperand.init ();
          myLastFiredPacket = null;
       }
 
       public virtual void onTableClear (bool isLeft)
       {
-         if (myLastFiredPacket is null) {
-            return;
-         }
-         requireSchema();
          //incase of natural join, left or right wont matter, the end result zeroes out
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinTable (mySchema);
@@ -122,19 +112,15 @@ namespace BCG
 
       public virtual void deleteRows (IBlueRowCollection rows, bool isLeft)
       {
-         var last = myLastFiredPacket;
-         if (last is null) {
-            return;
-         }
          JoinOperand thisOp, otherOp;
          int joinColOrdinal;
          myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft);
          foreach (IBlueRow row in rows)
          {
             thisOp.deleteRow (row);
-            if (otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)).Count == 0)
+            if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0)
                continue;   //row does not participate in join
-            ((JoinTable)last.newRows).deleteRow(isLeft, row, last);
+            ((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket);
          }
       }
 
@@ -149,12 +135,10 @@ namespace BCG
             otherOp = myLeftOperand;
             joinColOrdinal = myRightJoinColOrdinal;
          }
-         var schema = requireSchema();
-         myLastFiredPacket ??= new DataPacket(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema));
          foreach (IBlueRow row in rows)
          {
             thisOp.addRow (row);
-            RowList matchedNodes = otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal));
+            RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]);
             if (matchedNodes.Count == 0)
                continue;   //this row did not participate in the join
             if (isLeft)
@@ -162,22 +146,10 @@ namespace BCG
                   ((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket);
             else
                foreach (RowNode matchedNode in matchedNodes)
-                  ((JoinTable)myLastFiredPacket.newRows).addRow(ExceptionAssert.NotNull(matchedNode.row), row, myLastFiredPacket);
+                  ((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket);
          }
       }
 
-      [MemberNotNull(nameof(mySchema))]
-      protected JoinTableSchema requireSchema() =>
-         ExceptionAssert.NotNull(mySchema);
-
-      [MemberNotNull(nameof(myLastFiredPacket))]
-      protected DataPacket requireLastFiredPacket() {
-         if (myLastFiredPacket is null) {
-            myInitFiredPacket();
-         }
-         return myLastFiredPacket;
-      }
-
       private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft)
       {
          if (isLeft)
@@ -194,9 +166,6 @@ namespace BCG
          }
       }
 
-      object getJoinKey(IBlueRow row, int joinColOrdinal) =>
-         row[joinColOrdinal] ?? throw new ArgumentException("Row has null join key", nameof(row));
-
       /**************************************TEST METHOD(S)*********************************************/
       /// <summary>
       /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -204,7 +173,8 @@ namespace BCG
       /// </summary>
       /// <param name="leftPacket"></param>
       /// <param name="rightPacket"></param>
-      public void test(IDataPacket dp, bool isLeft) {
+      public void test (DataPacket dp, bool isLeft)
+      {
          JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
          jo.changeHandler (dp);
       }

src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs

@@ -1,6 +1,5 @@
-using Spark.Core.Diagnostics;
-
-using System.Collections.Generic;
+using System;
+using System.Collections;
 
 namespace BCG
 {
@@ -9,7 +8,7 @@ namespace BCG
 	/// </summary>
    internal class JoinDeletionIndex
    {
-      Dictionary<object, RowList> myIndex = new();
+      IDictionary myIndex = new Hashtable ();
       private string myOpIdName ;
       private bool myIsLeft;
 
@@ -29,8 +28,10 @@ namespace BCG
       /// <param name="row">JoinRow to add</param>
       public void add (JoinRow row)
       {
-         object o = ExceptionAssert.NotNull(row[myOpIdName]);
-         RowNode nodeAdded = myIndex.GetOrAdd(o, _ => new()).add(row);
+         object o = row[myOpIdName];
+         if (myIndex[o] == null)
+            myIndex[o] = new RowList ();
+         RowNode nodeAdded = ((RowList)myIndex[o]).add (row);
          if (myIsLeft)
             row.leftNode = nodeAdded;
          else
@@ -42,10 +43,7 @@ namespace BCG
       /// To remove a node, if node does not exist exception is thrown
       /// </summary>
       /// <param name="node"></param>
-      public void remove (RowNode node) {
-         var nodeId = ExceptionAssert.NotNull(node.row?[myOpIdName]);
-         myIndex[nodeId].remove(node);
-      }
+      public void remove (RowNode node)   {((RowList)myIndex[node.row[myOpIdName]]).remove (node);}
 
       public RowList getMatchingRows (object key)  {
          RowList retVal = (RowList)myIndex[key];

src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs

@@ -1,6 +1,7 @@
-using bts;
+using System;
+using BCG.Utilities;
 
-using Bts.DataStreamCore;
+using bts;
 
 namespace BCG
 {
@@ -8,11 +9,11 @@ namespace BCG
 	{
 		public JoinForeignTable (JoinTableSchema schema) : base (schema) {}
 
-      public override void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
-         var newRow = new JoinRow(leftRow, rightRow, mySchema);
-         int leftId = leftRow.Id();
-         Log.assert(myIdIndex[leftId] == null);
-         myIdIndex[leftId] = newRow;
+      public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
+      {
+         JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema);
+         Log.assert(myIdIndex[leftRow["id"]] == null);
+         myIdIndex[leftRow["id"]] = newRow;
          ((BlueTable)(dp.addedRows)).addRow (newRow);
          myLeftIndex.add (newRow);
          myRightIndex.add (newRow);

src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs

@@ -1,12 +1,11 @@
+using System;
+using System.Collections;
+using BCG.Utilities;
+
 using bts;
 
 using Bts.ColumnMapExpressions;
 
-using Spark.Core.Diagnostics;
-
-using System;
-using System.Collections;
-
 namespace BCG
 {
 	/// <summary>
@@ -38,8 +37,7 @@ namespace BCG
       private JoinTableSchema myJoinSchema;
       private IBlueTableSchema myLeftSchema, myRightSchema;
       private string myLeftColExpression, myRightColExpression;
-      string[] myUserRules;
-      string[]? myResolvedNames;
+      private string[] myUserRules, myResolvedNames;
       private IDictionary myFinalMap = new Hashtable ();
 
 
@@ -89,18 +87,16 @@ namespace BCG
       /// </summary>
       private void myFormUserRules ()
       {
-         var leftExp = ColumnMapExpression.Parse(myLeftColExpression) ?? throw new InvalidOperationException("Failed to parse left column expression");
-         var rightExp = ColumnMapExpression.Parse(myRightColExpression) ?? throw new InvalidOperationException("Failed to parse right column expression");
          if (DataStreamLoadFileConfig.instance.useStaticParser)
          {
-            myFormUserRules(true, leftExp);
-            myFormUserRules(false, rightExp);
+            myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
+            myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
          }
          else
          {
             // TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
-            myFormUserRules(true, leftExp);
-            myFormUserRules(false, rightExp);
+            myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
+            myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
          }
 
       }
@@ -119,7 +115,13 @@ namespace BCG
                OriginalNameInfo oni = myOriginalColInfo[i];
                if (oni.isLeft == isLeft)
                {
-                  myUserRules[i] = userMap[oni.colName] as string ?? oni.colName;
+                  if (userMap[oni.colName] != null)
+                  {
+                     if ((string)userMap[oni.colName] != oni.colName)
+                        myUserRules[i] = (string)userMap[oni.colName];
+                     else
+                        myUserRules[i] = oni.colName;
+                  }
                }
             }
          else
@@ -129,13 +131,10 @@ namespace BCG
                OriginalNameInfo oni = myOriginalColInfo[i];
                if (oni.isLeft == isLeft)
                {
-                  var colName = userMap[oni.colName] as string;
-                  // myUserRules[i] = userMap[oni.colName] as string 
-                  if (colName is null) {
+                  if (userMap[oni.colName] == null)
                      myUserRules[i] = oni.colName;
-                  } else if (colName != oni.colName) {
-                     myUserRules[i] = colName;
-                  }
+                  else if ((string)userMap[oni.colName] != oni.colName)
+                     myUserRules[i] = (string)userMap[oni.colName];
                }
             }
          }
@@ -166,12 +165,26 @@ namespace BCG
 
       private void myFormFinalMap ()
       {
-         ExceptionAssert.NotNull(myResolvedNames);
          foreach (string resolvedName in myResolvedNames)
             if (resolvedName != null)
                myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)];
       }
 
+
+      /// <summary>
+      /// Utility method to invert a hashtable
+      /// </summary>
+      /// <param name="arg"></param>
+      /// <returns></returns>
+   
+      private IDictionary reciprocateDictionary (IDictionary arg)
+      {
+         IDictionary retVal = new Hashtable ();
+         foreach (DictionaryEntry de in arg)
+            retVal[de.Value] = de.Key;
+         return retVal;
+      }
+
       /// <summary>
       /// Utility method to find all occurances of an obkect in an IList
       /// </summary>
@@ -181,11 +194,9 @@ namespace BCG
       private ArrayList myFindAllInCollection (IList list, object obj)
       {
          ArrayList retVal = new ArrayList ();
-         for (int i = 0; i < list.Count; i++) {
-            if (Equals(list[i], obj)) {
+         for (int i = 0; i < list.Count; i++)
+            if (list[i] != null && list[i].Equals (obj))
                retVal.Add (i);
-            }
-         }
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs

@@ -1,7 +1,6 @@
-using Spark.Core.Diagnostics;
-
 using System;
-using System.Diagnostics.CodeAnalysis;
+using System.Collections;
+using BCG.Utilities;
 
 namespace BCG
 {
@@ -12,46 +11,34 @@ namespace BCG
    public class JoinOperand 
    {
       private string myJoinCol;
-      IDataPacket? myLastArrivedPacket;
-      JoinOperand? myOtherOperand;
+      private DataPacket myLastArrivedPacket;
+      private JoinOperand myOtherOperand;
       private bool myIsLeft;
-      JoinOperandRowIndex? myRowIndex;
-      InternalJoinStream? myMaster;
-
+      private JoinOperandRowIndex myRowIndex;
+      private InternalJoinStream myMaster;
+      private IBlueTableSchema myOpSchema;
+      
       public JoinOperand (string joinCol, bool isLeft)
       {
          myJoinCol = joinCol;
          myIsLeft = isLeft;
       }
 
-      [MemberNotNullWhen(true, nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
-      bool isInitialized {
-         get {
-            if (myLastArrivedPacket is null) {
-               return false;
-            }
-            ExceptionAssert.NotNull(myMaster, myLastArrivedPacket, operandSchema, myOtherOperand, myRowIndex);
-            return true;
-         }
-      }
-
       internal void changeHandler (object o)
       {
-         var dp = (IDataPacket)o;
-         if (isInitialized) {
-            mySubsequentChangeHandler (dp);
-         } else {
-            myFirstChangeHandler(dp);
-         }
+         DataPacket temp = (DataPacket)o;
+         if (myLastArrivedPacket != null)
+            mySubsequentChangeHandler ((DataPacket)o);
+         else
+            myFirstChangeHandler ((DataPacket)o);
       }
 
-      [MemberNotNull(nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
-      bool myFirstChangeHandler(IDataPacket arrivedDp) {
-         ExceptionAssert.NotNull(myMaster);
+      private void myFirstChangeHandler (DataPacket arrivedDp)
+      {
          myLastArrivedPacket = arrivedDp;
-         operandSchema = myLastArrivedPacket.newRows.schema;
+         myOpSchema = myLastArrivedPacket.newRows.schema;
          myOtherOperand = myMaster.getOtherOperand (this);
-         myRowIndex = new JoinOperandRowIndex (operandSchema[myJoinCol].colNo, operandSchema["id"].colNo);
+         myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo);
          if (myOtherOperand.hasReceivedData)
          {
             //the other operand has already received data, so the join would
@@ -62,16 +49,14 @@ namespace BCG
             myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft);
             myMaster.endSession ();
          }
-         else {
+         else
             myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows, 
-               operandSchema[myJoinCol].colNo,
-               operandSchema["id"].colNo);
-         }
-         return true;
+               myOpSchema[myJoinCol].colNo,
+               myOpSchema["id"].colNo);
       }
 
-      void mySubsequentChangeHandler (IDataPacket arrivedDp) {
-         RequireInitialized();
+      private void mySubsequentChangeHandler (DataPacket arrivedDp)
+      {
          myLastArrivedPacket = arrivedDp;
          if (!myOtherOperand.hasReceivedData)
          {
@@ -100,46 +85,29 @@ namespace BCG
          myMaster.endSession ();
       }
 
-      internal void addRow (IBlueRow row) =>
-         ExceptionAssert.NotNull(myRowIndex).add(row);
+      internal void addRow (IBlueRow row) {myRowIndex.add (row);}
 
-      internal void deleteRow (IBlueRow row) {
-         RequireInitialized();
-         myRowIndex.remove (row);
-      }
+      internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);}
 
-      internal IBlueTableSchema? operandSchema { get; private set; }
+      internal IBlueTableSchema operandSchema  {get{return myOpSchema;}}
 
       internal InternalJoinStream master {set{myMaster = value;}}
 
       internal string joinCol   {get{return myJoinCol;}}
 
-      internal int joinColOridinal {
-         get {
-            RequireInitialized();
-            return operandSchema[myJoinCol].colNo;
-         }
-      }
+      internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}}
   
-      internal RowList getMatchingRows (object key) {
-         RequireInitialized();
-         return myRowIndex.getMatchingRows(key);
-      }
+      internal RowList getMatchingRows (object key)   {return (RowList) myRowIndex.getMatchingRows (key);}
 
       internal bool hasReceivedData   {get{return myLastArrivedPacket != null;}}
 
-      internal void Reset() {
+      internal void init ()  
+      {
          ///should be brought back to the the state before the first callback 
          ///(ie the state after construction)
          myRowIndex = null;
          myLastArrivedPacket = null;
       }
-
-      [MemberNotNull(nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
-      internal void RequireInitialized() {
-         if (!isInitialized) {
-            throw new InvalidOperationException("Internal state not properly initialized");
-         }
-      }
+      
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs

@@ -1,6 +1,6 @@
-using Spark.Core.Diagnostics;
-
-using System.Collections.Generic;
+using System;
+using System.Collections;
+using BCG.Utilities;
 
 namespace BCG
 {
@@ -11,10 +11,8 @@ namespace BCG
 	/// </summary>
 	internal class JoinOperandRowIndex
 	{
-      readonly Dictionary<object, RowList> myJoinColDict = new();
-      readonly Dictionary<int, RowNode> myIdDict = new();
-      readonly int myJoinColOrdinal;
-      readonly int myIdColOrdinal;
+      private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
+      private int myJoinColOrdinal, myIdColOrdinal;
 
       /// <summary>
       /// 
@@ -36,14 +34,19 @@ namespace BCG
 
       public void add (IBlueRow row)
       {
-         myIdDict[getId(row)] = myJoinColDict.GetOrAdd(getJoinKey(row), _ => new()).add(row);
+         RowList list;
+         if (myJoinColDict[row[myJoinColOrdinal]] == null)
+            myJoinColDict[row[myJoinColOrdinal]] = list = new RowList (); 
+         else
+            list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
+         myIdDict[row[myIdColOrdinal]] = list.add (row);
       }
 
       public void remove (IBlueRow row)
       {
-         var id = getId(row);
-         myJoinColDict[getJoinKey(row)].remove(myIdDict[id]);
-         myIdDict.Remove(id);
+         RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
+         ((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
+         myIdDict.Remove (row[myIdColOrdinal]);
       }
 
       public RowList getMatchingRows (object key)  
@@ -59,11 +62,5 @@ namespace BCG
          myJoinColDict.Clear ();
          myIdDict.Clear ();
       }
-
-      object getJoinKey(IBlueRow row) =>
-         ExceptionAssert.NotNull(row[myJoinColOrdinal]);
-
-      int getId(IBlueRow row) =>
-         (int)row[myIdColOrdinal]!;
 	}
 }

src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs

@@ -1,21 +1,17 @@
-using bts;
-
-using Bts.DataStreamCore;
-
 using System;
+using BCG.Utilities;
 using System.Collections;
+using bts;
 
 namespace BCG
 {
    internal class JoinRow : IBlueRow 
    {
-      RowNode? myLeftNode;
-      RowNode? myRightNode;
-      protected IBlueRow myLeftRow;
-      protected IBlueRow? myRightRow;
+      private RowNode myLeftNode, myRightNode;
+      protected IBlueRow myLeftRow, myRightRow;
       protected int myId;
       protected JoinTableSchema mySchema;
-      private object?[] myValues;
+      private object[] myValues;
       
       #region Constructor(s)
 
@@ -25,8 +21,8 @@ namespace BCG
       /// <param name="leftRow">left row of the join</param>
       /// <param name="rightRow">right row of the join</param>
       /// <param name="schema">schema of the joined row</param>
-      public JoinRow (IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) 
-         : this(leftRow.Id(), leftRow, rightRow, schema) { }
+      public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) 
+         : this ((int)leftRow["id"], leftRow, rightRow, schema)   {}
 
       /// <summary>
       /// Constructs a row assigning it the ID provioded as the first argument
@@ -35,7 +31,8 @@ namespace BCG
       /// <param name="leftRow">left row of the join</param>
       /// <param name="rightRow">right row of the join</param>
       /// <param name="schema">schema of the joined row</param>
-      public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) {
+      public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
+      {
          myId = id;
          myLeftRow = leftRow;
          myRightRow = rightRow;
@@ -51,13 +48,13 @@ namespace BCG
 
       public bool isPartialJoin{get{return myRightRow == null;}}
 
-      public RowNode? leftNode 
+      public RowNode leftNode 
       {
          get{return myLeftNode;}
          set{myLeftNode = value;}
       }
 
-      public RowNode? rightNode
+      public RowNode rightNode
       {
          get{return myRightNode;}
          set{myRightNode = value;}
@@ -88,7 +85,7 @@ namespace BCG
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public virtual object? this[object key]
+      public virtual object this [object key]
       {
          get
          {
@@ -99,7 +96,7 @@ namespace BCG
          set {throw new NotSupportedException ("The IBlueRow is read only");}
       }
 
-      public object? this[int index]
+      public object this [int index]
       {
          get
          {
@@ -122,8 +119,7 @@ namespace BCG
          }
       }
 
-      public object? this[string colName] =>
-         this[mySchema[colName].colNo];
+      public object this [string colName] {get{return this[mySchema[colName].colNo];}}
    
       public ICollection Keys{get{return mySchema.columnNames;}}
 
@@ -137,8 +133,7 @@ namespace BCG
          }
       }
 
-      public void Add (object key, object? value) =>
-         throw new NotSupportedException ("The IBlueRow is readonly");
+      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
 
       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}