refactor: Add IDataStreamFactory(s) for spatial arb streams

It looks in the working directory for runtime config appsettings.json files.

.devcontainer/Dockerfile

@@ -55,16 +55,14 @@ RUN \
    clangd-$clang_major
 
 # CMake
-RUN set -eux \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends ca-certificates curl gpg \
-  && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
-    | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
-  && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-    > /etc/apt/sources.list.d/kitware.list \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends cmake \
-  && rm -rf /var/lib/apt/lists/*
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
 
 # Dotnet
 RUN apt-get update && apt-get install -y dotnet8
@@ -82,14 +80,9 @@ RUN mkdir -p /etc/apt/keyrings \
 # Emumba (deps)
 RUN apt-get update && apt-get -y install rapidjson-dev
 # Emumba
-ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
-RUN --mount=type=secret,id=bitbucket_access_token \
-  set -eux \
-    && apt-get update \
-    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
-    && curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
-    && apt-get install -y /tmp/emumba-deribit-connector.deb \
-    && rm -f /tmp/emumba-deribit-connector.deb
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
 
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -97,18 +90,6 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
-# Rakurai
-ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
-RUN --mount=type=secret,id=bitbucket_access_token \
-  set -eux \
-    && apt-get update \
-    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
-    && curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
-    # Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
-    # spdlog headers.
-    && apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
-    && rm -f /tmp/rakuirai.deb
-
 # Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \

.devcontainer/devcontainer.json

@@ -1,11 +1,5 @@
 {
-  "build": {
-    "dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
-    "options": [
-      "--secret",
-      "id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
-    ]
-  },
+  "dockerFile": "Dockerfile",
   "hostRequirements": {
     "memory": "8gb",
     "storage": "10gb"

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.env

@@ -29,9 +29,6 @@
 #     tools/deploy.sh --help
 #
 
-### Host/Runtime ###
-TZ=Etc/UTC
-
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
 #   src/server/sparkd/Dockerfile
@@ -41,9 +38,6 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
-# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
-# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
-# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
@@ -51,14 +45,5 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
-SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
 
-### PubSubServer ###
-PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
-PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-
-### MdPublisher ###
-MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
-MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -55,8 +55,6 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
-/src/**/*appsettings.Local.json
-/src/**/*/Configurations/Local/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/

.vscode/launch.json

@@ -19,19 +19,15 @@
       "type": "coreclr",
       "request": "launch",
       "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
       "windows": {
+        "preLaunchTask": "Build Api Server (Debug)",
         "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        "args": [
+          "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
+        ],
+        "cwd": "build/Debug/bin/net8.0",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -43,20 +39,43 @@
       "program": "",
       "windows": {
         "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        "args": [
+          "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
+        ],
+        "cwd": "build/Debug/bin/net8.0",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "Bts.DataStreamWebApp.Server",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "env": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
+        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "env": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
+        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -142,14 +161,14 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
           "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -164,8 +183,8 @@
         "program": "DataStreamQueryAnalyzer.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
           "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -208,43 +227,6 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
-    {
-      "name": "Sdk Demo",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "preLaunchTask": "Build Spark Sdk (Debug)",
-      "windows": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "linux": {
-        "program": "Bts.Spark.Sdk.Demo",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sdk Demo (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "linux": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
     {
       "name": "Spark",
       "type": "coreclr",
@@ -285,7 +267,7 @@
       "windows": {
         "program": "Spark.exe",
         "args": [
-          "--Spark:ConfigServer",
+          "--sparkd",
           "${input:sparkConfigServer}"
         ],
         "cwd": "build/Debug/bin/net8.0-windows",
@@ -333,7 +315,6 @@
       "program": "",
       "linux": {
         "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-        "preLaunchTask": "Build Sparkd (Debug | Clang)",
       },
       "args": [
         "run"
@@ -383,49 +364,44 @@
         }
       ]
     },
-    {
-      "name": "Sparkd Attach",
-      "type": "cppdbg",
-      "request": "attach",
-      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
   ],
   "inputs": [
     {
-      "id": "apiServerEnvironment",
+      "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
-      "description": "Api Server Configuration",
+      "description": "Publisher Configuration",
       "options": [
         "Development",
-        "Local",
+        "MarketData",
+        "Production",
+        "ProductionAws",
+        "Staging",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerEnvironment",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "Development",
+        "EchoServer",
+        "Production",
+        "ProductionAws",
         "Staging",
       ],
       "default": "Development",
     },
     {
-      "id": "dataStreamPublisherConfig",
+      "id": "dataStreamWebAppServerEnvironment",
       "type": "pickString",
-      "description": "Publisher Configuration",
+      "description": "DataStream WebApp Environment",
       "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
-      ],
-      "default": "MarketDataPublisher",
+        "Development",
+        "Production",
+        "ProductionAws",
+        "Staging",
+      ]
     },
     {
       "id": "dataStreamQueryAnalyzerEnvironment",
@@ -433,48 +409,9 @@
       "description": "DataStreamQueryAnalyzer Configuration",
       "options": [
         "Development",
-        "Local",
       ],
       "default": "Development",
     },
-    {
-      "id": "dataStreamPublisherEnvironment",
-      "type": "pickString",
-      "description": "DataStreamPublisher Configuration",
-      "options": [
-        "Local",
-        "ReplayMarketData",
-      ],
-      "default": "Local",
-    },
-    {
-      "id": "pubSubServerEnvironment",
-      "type": "pickString",
-      "description": "PubSubServer Configuration",
-      "options": [
-        "Development",
-        "EchoServer",
-        "Local",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "sdkDemoEnvironment",
-      "type": "pickString",
-      "description": "Sdk Demo Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "sparkConfigServer",
-      "type": "promptString",
-      "description": "Config Server URL",
-      "default": "http://localhost:8000/config",
-    },
     {
       "id": "sparkEnvironment",
       "type": "pickString",
@@ -485,6 +422,12 @@
       ],
       "default": "Development",
     },
+    {
+      "id": "sparkConfigServer",
+      "type": "promptString",
+      "description": "Config Server URL",
+      "default": "http://localhost:8000/config",
+    },
     {
       "id": "sparkTest",
       "type": "promptString",
@@ -495,5 +438,15 @@
       "type": "promptString",
       "description": "Test to run",
     },
+    {
+      "id": "isSparkApiEnabled",
+      "type": "pickString",
+      "description": "Enable Spark Api (connect to sparkd)",
+      "options": [
+        "True",
+        "False",
+      ],
+      "default": "True",
+    }
   ]
 }

.vscode/python.env

@@ -1 +1 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins

.vscode/tasks.json

@@ -121,6 +121,117 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Build DataStreamQueryAnalyzer",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Build Spark",
       "group": "build",
@@ -172,66 +283,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Build Spark Sdk",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark Sdk (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark Sdk (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Build Sparkd",
       "group": "build",
@@ -363,6 +414,22 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Publish DataStreamWebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/publish_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/publish_data_stream_web_app.cmd",
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Edit DataStream Config",
       "type": "process",
@@ -411,141 +478,19 @@
         "id": "wrench"
       },
     },
-    {
-      "label": "Run Api Server",
-      "dependsOn": "Build Api Server",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug)",
-      "dependsOn": "Build Api Server (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release)",
-      "dependsOn": "Build Api Server (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
     {
       "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
+      "dependsOn": "Build DataStreamQueryAnalyzer",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
         },
       },
       "icon": {
@@ -555,16 +500,17 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
         },
       },
       "icon": {
@@ -574,16 +520,17 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
       "type": "process",
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
         },
       },
       "icon": {
@@ -657,6 +604,89 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run DataStream WebApp (Debug)",
+      "dependsOn": "Build DataStream WebApp (Debug)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Development",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStream WebApp (Release)",
+      "dependsOn": "Build DataStream WebApp (Release)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Production",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStream WebApp (Release | Skip Build)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Production",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run PubSubServer",
       "dependsOn": "Build DataStream",
@@ -1024,17 +1054,6 @@
     },
   ],
   "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "buildConfiguration",
       "type": "pickString",
@@ -1045,16 +1064,6 @@
       ],
       "default": "Debug",
     },
-    {
-      "id": "dataStreamQueryAnalyzerEnvironment",
-      "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
     {
       "id": "sparkdCmakePreset",
       "type": "pickString",

Bts.Spark.Sdk.sln

@@ -1,102 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|Any CPU = Release|Any CPU
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.Build.0 = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.Build.0 = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.Build.0 = Release|x64
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.Build.0 = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-	EndGlobalSection
-EndGlobal

CMakePresets.json

@@ -8,31 +8,14 @@
 
   "configurePresets": [
     {
-      "name": "spark-base",
-      "hidden": true,
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
-      }
-    },
-
-    {
-      "name": "spark-windows",
-      "inherits": "spark-base",
-      "displayName": "Spark",
+      "name": "spark",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
+      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
@@ -40,9 +23,18 @@
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
-        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
       }
     },
 
@@ -159,17 +151,17 @@
     },
 
     {
-      "name": "spark-linux",
-      "inherits": "spark-base",
-      "displayName": "Spark",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
+      "name": "spark-api-server-windows",
+      "inherits": "spark",
+      "displayName": "Spark Api Server (Windows)"
+    },
+
+    {
+      "name": "spark-api-server-linux",
+      "inherits": "sparkd",
+      "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "platform": "linux",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -178,26 +170,21 @@
           "type": "BOOL",
           "value": "OFF"
         },
-        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_EXPORT_COMPILE_COMMANDS": {
           "type": "BOOL",
           "value": "ON"
         },
-        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
-        "DATABENTO_USE_EXTERNAL_JSON": {
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
+        "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "EXTERNALS_ROOT": "${sourceDir}/externals/linux"
+        }
       }
     },
 
@@ -217,32 +204,87 @@
       "cacheVariables": {
         "CCAPI_LOG_LEVEL": "Trace"
       }
-    }
-  ],
+    },
 
-  "buildPresets": [
     {
-      "name": "spark-windows",
+      "name": "data-stream-suite-windows",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
-      "hidden": true,
-      "configurePreset": "spark-windows"
+      "inherits": "spark",
+      "displayName": "Data Stream Suite (Windows)",
+      "description": "Data Stream utils ported over from Edge",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
+      }
     },
 
     {
-      "name": "spark-windows-debug",
+      "name": "data-stream-suite-linux",
+      "displayName": "Data Stream Suite (Linux)",
+      "description": "Data Stream utils ported over from Edge",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "inherits": "sparkd",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    }
+  ],
+
+  "buildPresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true
+    },
+
+    {
+      "name": "spark-debug",
       "displayName": "Spark (Debug)",
-      "inherits": "spark-windows",
+      "inherits": "spark",
+      "configurePreset": "spark",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-windows-release",
+      "name": "spark-release",
       "displayName": "Spark (Release)",
-      "inherits": "spark-windows",
+      "inherits": "spark",
+      "configurePreset": "spark",
       "configuration": "Release"
     },
 
@@ -285,27 +327,52 @@
     },
 
     {
-      "name": "spark-linux",
+      "name": "spark-api-server-windows",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true,
+      "configurePreset": "spark-api-server-windows"
+    },
+
+    {
+      "name": "spark-api-server-windows-debug",
+      "displayName": "Spark Api Server (Windows | Debug)",
+      "inherits": "spark-api-server-windows",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "spark-api-server-windows-release",
+      "displayName": "Spark Api Server (Windows | Release)",
+      "inherits": "spark-api-server-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "spark-api-server-linux",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
       "hidden": true,
-      "configurePreset": "spark-linux"
+      "configurePreset": "spark-api-server-linux"
     },
 
     {
-      "name": "spark-linux-debug",
-      "inherits": "spark-linux",
-      "displayName": "Spark (Linux | Debug)",
+      "name": "spark-api-server-linux-debug",
+      "inherits": "spark-api-server-linux",
+      "displayName": "Spark Api Server (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-linux-release",
-      "inherits": "spark-linux",
-      "displayName": "Spark (Linux | Release)",
+      "name": "spark-api-server-linux-release",
+      "inherits": "spark-api-server-linux",
+      "displayName": "Spark Api Server (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -322,6 +389,34 @@
       "configurePreset": "sparkd-ccapi-diag",
       "displayName": "Debug (ccapi diag)",
       "inherits": "sparkd"
+    },
+
+    {
+      "name": "data-stream-suite-windows-debug",
+      "displayName": "Data Stream Suite (Windows | Debug)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-windows-release",
+      "displayName": "Data Stream Suite (Windows | Release)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "data-stream-suite-linux-debug",
+      "displayName": "Data Stream Suite (Linux | Debug)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-linux-release",
+      "displayName": "Data Stream Suite (Linux | Release)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Release"
     }
   ]
 }

Spark.Api.Server.sln

@@ -1,11 +1,10 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
@@ -43,538 +42,168 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
-		MinSizeRel|Any CPU = MinSizeRel|Any CPU
-		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
-		Release|Any CPU = Release|Any CPU
-		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
-		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
-		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.Build.0 = Debug|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.ActiveCfg = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.Build.0 = Release|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.Build.0 = Debug|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.ActiveCfg = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.Build.0 = Release|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.Build.0 = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.Build.0 = Release|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.Build.0 = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.Build.0 = Release|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.Build.0 = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.Build.0 = Release|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.Build.0 = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.Build.0 = Release|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.Build.0 = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.Build.0 = Release|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.Build.0 = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.Build.0 = Release|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.Build.0 = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.Build.0 = Release|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.Build.0 = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.Build.0 = Release|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.Build.0 = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.Build.0 = Release|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.Build.0 = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.Build.0 = Release|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
@@ -586,17 +215,8 @@ Global
 		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
 	EndGlobalSection
 EndGlobal

cmake/Protobuf.cmake

@@ -0,0 +1,182 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE}
+          ARGS
+            --cpp_out=${_ARGS_CXX_OUT_DIR}
+            --plugin=protoc-gen-service=${_plugin_cmd}
+            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
+            -I ${PROTOBUF_INCLUDE_DIRS}
+            -I ${_protobuf_src_dir}
+            ${ABS_FIL}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_cmd}
+          ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM
+      )
+    else()
+      add_custom_command(
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM
+      )
+    endif()
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+

compose.yaml

@@ -1,8 +1,7 @@
 services:
   sparkd:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -10,35 +9,24 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-shell:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-    build:
-      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: "do sleep 1; done"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
+        ./sparkd run"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -53,9 +41,8 @@ services:
 
   sparkd-symbols:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -63,8 +50,26 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
+        ./sparkd run"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -72,7 +77,6 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       restart_policy:
         condition: any
@@ -80,75 +84,17 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-  sparkd-multicast:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
-  
-  sparkd-multicast-cert:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
+
   sparkd-tail:
-    image: bluetradingsystems/linux_dotnet8
+    image: bluetradingsystems/linux_dotnet7
     volumes:
       - type: volume
         read_only: true
         source: sparkd_data
-        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
+        target: /home/bts/sparkdata/data/logging.sparkd
         volume:
           subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
 
   sparkd-ping-test:
     depends_on:
@@ -158,75 +104,35 @@ services:
     build:
       context: .
       dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet8
+        FROM bluetradingsystems/linux_dotnet7
         USER "root"
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
 
   pubsub:
-    volumes:
-      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
+    environment:
+      - DOTNET_ENVIRONMENT=Staging
     build:
       dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
       args:
-        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
-        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
     ports:
       - 22019:22019
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
-        if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
-          mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/pubsub/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/pubsub/conf/.env.local
-          set +a
-        fi
-        ./PubSubServer"
-    env_file:
-      - .env
-      - ${PUBSUB__CONFIG_DIR}/.env
+    command: [ "./PubSubServer" ]
 
   md-publisher:
-    volumes:
-      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
-      - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
     build:
       dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
       args:
-        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
-        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
-        if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
-          mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/md_publisher/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/md_publisher/conf/.env.local
-          set +a
-        fi
-      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
-    env_file:
-      - .env
-      - ${MD_PUBLISHER__CONFIG_DIR}/.env
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+    depends_on:
+      - pubsub
+      - sparkd
+    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
+    environment:
+      - DOTNET_ENVIRONMENT=MarketData
 
   api-server:
     depends_on:
@@ -237,18 +143,21 @@ services:
     build:
       dockerfile: src/client/spark_api_server/Dockerfile
     command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+
+  webapp:
+    ports:
+      - 8080:8080
+      - 8443:8443
+    build:
+      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
+    command: [ "./Bts.DataStreamWebApp.Server" ]
     environment:
       - ASPNETCORE_ENVIRONMENT=Staging
 
 volumes:
   sparkd_data:
     name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
-  pubsub_data:
-    name: "pubsub_22019"
-  md_publisher_data:
-    name: "md_publisher_22019"
-  historic_md:
-    name: "historic_md"
-  md_recording:
-    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
-    external: true

doc/sparkd-security-id-lifecycle.md

@@ -1,942 +0,0 @@
-# Security ID handling in `sparkd`
-
-This document traces how security identifiers are created, translated, persisted,
-indexed, and published by the `sparkd` market-data paths. It focuses on the
-legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
-
-## Executive summary
-
-`sparkd` uses two identifiers that must not be confused:
-
-| Identifier | Meaning | Typical field | Lifetime |
-|---|---|---|---|
-| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
-| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
-
-`invalid_security_key` is `0`; a security with key `0` is not yet publicly
-discoverable through the shared-memory consumer path.
-
-The two principal paths differ:
-
-- **MDP3/OnixS:** the definition callback sets only the CME ID in
-  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
-  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
-  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
-  calls `MDProducerHandle::set_key()`.
-- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
-  currently uses the ID already present on the incoming domain security
-  (`use_given_sec_key == true`). Live adapters normally obtain that ID
-  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
-  it to both `FixedSecurity::key` and `numeric_id`, so
-  `MDProducer::add_security()` publishes it synchronously. Only the listener's
-  own fallback `IDGenerator` call is bypassed by the current constant.
-
-## End-to-end diagrams
-
-### CME MDP3 / OnixS
-
-```mermaid
-sequenceDiagram
-    participant FE as OnixS feed-engine thread
-    participant L as MDP3Listener
-    participant P as MDProducer / shared memory
-    participant Q as SecurityIDRecorder queues
-    participant D as sparkd dispatcher thread
-    participant DB as key_ids.sqlite
-
-    FE->>L: instrument definition (CME SecurityID)
-    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
-    L->>P: add_security(FixedSecurity)
-    P->>P: allocate hot + cold entries with key 0
-    P->>Q: ensure_key(handle)
-    L->>L: index[CME SecurityID] = handle
-    Note over FE,L: Books can route by CME ID while the Spark key is pending
-    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
-    Q->>DB: find (namespace, descriptor), or insert a new row
-    DB-->>Q: SQLite rowid = Spark security key
-    Q->>P: handle.set_key(rowid)
-    P->>P: hot.key first, then cold security.key, then increment security_count
-    Note over P: The security is now public to shared-memory consumers
-```
-
-### Generalized/newer pipeline (current behavior)
-
-```mermaid
-sequenceDiagram
-    participant A as Venue adapter / plugin
-    participant DB as IDGenerator / key_ids.sqlite
-    participant G as GeneralMarketDataListener
-    participant I as GeneralSecurityIndex
-    participant P as MDProducer / shared memory
-
-    A->>DB: get_or_add(venue, type, adapter key)
-    DB-->>A: SQLite rowid as domain SecurityID
-    A->>G: on_securities(domain Security list with ID)
-    G->>G: _find_sec() - use incoming security_common.id
-    G->>G: _copy_security_common() - key = id, numeric_id = id
-    G->>P: add_security(FixedSecurity with non-zero key)
-    P->>P: set_key() during add_security()
-    G->>I: index[id] = handle
-    Note over G,P: No asynchronous ID assignment in the current configuration
-```
-
-## Data-flow charts
-
-The sequence diagrams above emphasize timing and thread transitions. These
-flowcharts emphasize which identifier is carried through each representation
-and where data is retained.
-
-### MDP3 / legacy data flow
-
-```mermaid
-flowchart TD
-    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
-
-    ONIXS --> DEF["MDP3Listener definition callback"]
-    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
-
-    FIXED --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
-    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
-
-    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
-
-    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
-    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
-    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
-    UNRESOLVED --> CMEINDEX
-    CMEINDEX --> LEGS
-    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
-
-    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
-    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
-    INSERT --> ROWID
-
-    ROWID --> SETKEY["MDProducerHandle::set_key"]
-    SETKEY --> HOT["Write hot-entry key"]
-    HOT --> COLD["Write cold security key"]
-    COLD --> COUNT["Increment security_count"]
-    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
-
-    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
-    UPDATE --> CMEINDEX
-    CMEINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> LIVE["Update live shared-memory data"]
-    PUBLIC --> LIVE
-```
-
-The CME index is usable before the Spark key is assigned, so feed updates can
-reach the internal handle while the security is still hidden from consumers.
-The persistent rowid crosses the publication boundary only at `set_key()`.
-
-### Generalized/newer data flow
-
-```mermaid
-flowchart TD
-    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
-
-    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
-    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
-    IDINPUT --> MUTEX["IDGenerator global mutex"]
-    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid"]
-    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
-    INSERT --> ROWID
-    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
-    IDTYPE -- Recorded or pre-keyed --> DOMAINID
-
-    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
-    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
-    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
-    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
-
-    CONVERT --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
-    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
-    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
-
-    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
-    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
-    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
-    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
-    PYHOT --> EVENTID
-    PYHOT -. uses same mutex and SQLite path .-> MUTEX
-    EVENTID --> CCMAP
-    CCMAP --> GENERALINDEX
-    GENERALINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
-    SHM --> SHMUPDATE
-
-```
-
-In the normal newer live path, persistence occurs before the definition enters
-the generalized listener. The DB call therefore adds synchronous adapter-side
-latency, but publication needs no later timer. Most adapters use an in-memory
-map for subsequent events; the dotted Python path is the documented exception.
-
-## The publication boundary
-
-`MDProducerHandle::set_key()` is the definitive publication point for the
-legacy shared-memory representation. Its write ordering is intentional:
-
-1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
-2. Write the key into the cold security entry.
-3. Increment the cold header's public `security_count`.
-
-The cold security key is written last because consumers treat a non-zero cold
-key as visibility. At that moment the corresponding hot entry must already be
-addressable. `set_key()` only permits the transition from `0` to one key; it is
-not a key-update mechanism.
-
-## Identifier representations
-
-The names are historically inconsistent. In particular, many protobuf fields
-named `security_id` contain the **Spark key**, not the exchange's ID.
-
-| Layer | Exchange/feed identifier | Spark identifier |
-|---|---|---|
-| MDP3 wire/OnixS | `securityId()` | not present |
-| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
-| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
-| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
-| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
-| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
-| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
-| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
-
-The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
-shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
-64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
-allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
-no explicit checked conversion.
-
-## Which parts use exchange IDs, descriptors, or Spark security IDs?
-
-The three identifier classes serve different jobs:
-
-- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
-- **Descriptor/natural key:** recognizes a definition logically at the
-  persistence, discovery, or human-facing boundary.
-- **Spark security ID/key:** joins the instrument to books, orders, fills,
-  positions, and services after publication.
-
-The core downstream system does not repeatedly identify a security from its
-description. The description is primarily used to obtain a Spark key; after
-that boundary, most joins use the numeric Spark key.
-
-### Component-level usage matrix
-
-| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
-|---|---|---|---|
-| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
-| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
-| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
-| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
-| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
-| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
-| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
-| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
-| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
-| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
-| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
-| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
-| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
-| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
-| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
-| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
-| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
-| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
-| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
-| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
-
-### Exchange/external IDs: where they stop
-
-In MDP3 the separation is explicit:
-
-```text
-CME SecurityID
-    -> FixedSecurity.numeric_id
-    -> CMESecurityIndex lookup
-    -> MDProducerHandle
-    -> Spark security_key for downstream publication
-```
-
-The external CME ID continues to route feed messages but does not become the
-order/position key. This permits the same downstream Spark key to survive a
-restart as long as the logical descriptor mapping remains in
-`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
-
-The generalized path collapses more of these roles:
-
-```text
-exchange/vendor identity
-    -> adapter calls IDGenerator
-    -> domain::SecurityID
-    -> FixedSecurity.key and FixedSecurity.numeric_id
-    -> GeneralSecurityIndex and downstream Spark key
-```
-
-`SecurityCommon::exchange_sec_id` may still contain the original textual
-external identifier, but `GeneralMarketDataListener` does not use it to route
-events and does not put it in `numeric_id`. The adapter must translate later
-events to `domain::SecurityID` before invoking the listener.
-
-### Descriptor/name users
-
-Descriptors are used in three main places:
-
-1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
-   descriptor, while `IDGenerator` uses its caller-provided `name`.
-2. **Definition lookup and discovery:** `SecurityCache` supports
-   venue/type/name lookup; symbol filters and administrative tools use security
-   metadata to select or display instruments.
-3. **Diagnostics:** logs include symbols and descriptions so humans can
-   understand a numeric key.
-
-Descriptions generally do not drive the high-rate book, trade, order, fill, or
-position paths. Those paths use numeric IDs after definition resolution.
-
-It is important that “descriptor” does not mean one universal canonical field:
-
-- Legacy CME uses Globex `security_desc` from tag 55.
-- Many generalized adapters use `sec.desc()` or an exchange instrument name.
-- Databento uses its numeric instrument ID converted to a string.
-- Python uses `exchange_security_id`.
-
-Therefore, two adapters may describe the same economic contract but request
-different logical rows and receive different Spark IDs.
-
-### Spark security-ID users
-
-Once a security is published, its Spark key is the dominant system identifier:
-
-```mermaid
-flowchart LR
-    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
-    KEY --> MDC["MDConsumer and book subscriptions"]
-    KEY --> MBO["MBO snapshots and queries"]
-    KEY --> OMS["Orders, algorithms, and spread legs"]
-    KEY --> FILLS["Fills and execution records"]
-    KEY --> POS["Positions, PNL, and risk"]
-    KEY --> API["Services and client APIs"]
-    KEY --> REC["Relay and market-data recordings"]
-    KEY --> SECDB["Serialized security definitions"]
-```
-
-Fields named `security_id`, `security_key`, or top-level `Security.id()` in
-these downstream areas almost always mean the Spark key. They should not be
-compared directly with CME `SecurityID`, Databento `instrument_id`, or another
-venue/vendor identifier without going through the relevant adapter/index
-mapping.
-
-## Persistence and allocation
-
-### The authoritative key mapping: `key_ids.sqlite`
-
-Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
-file is:
-
-```text
-<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
-```
-
-The database contains one table and two indexes:
-
-```sql
-CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
-CREATE UNIQUE INDEX key_index ON store(ns, key);
-CREATE INDEX ns_index ON store(ns);
-```
-
-The SQLite `rowid` is the generated Spark key. `ns` is normally
-`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
-descriptor chosen by the caller. `value` is normally empty. Therefore ID
-stability depends on all three of these remaining stable:
-
-- the configured `data_path` and preservation of this database;
-- the venue/type namespace;
-- the descriptor supplied by the feed adapter.
-
-The two allocator implementations share the file and convention:
-
-- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
-  uses `securities::descriptor()`. For CME this is the Globex
-  `security_desc`, populated from the definition's symbol/tag 55.
-- `IDGenerator` (newer): derives the namespace from new domain enums and uses
-  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
-  identifier; Databento deliberately passes the decimal instrument ID, and the
-  Python bridge uses the exchange security ID.
-
-### Logical-row examples
-
-The following names are illustrative. Production uses the exact string emitted
-by the selected feed/adapter, including its case, spacing, and weekly-series
-notation.
-
-#### Legacy `SecurityIDRecorder`
-
-For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
-Globex `security_desc`. After converting the fixed security to
-`api::Security`, `securities::descriptor()` returns that same
-`glbx.security_desc()` string.
-
-| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
-|---|---|---|---|
-| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
-| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
-| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
-| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
-
-A weekly option does not get a special namespace. Standard and weekly options
-are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
-`security_desc` distinguishes the weekly series. For example, if the database
-assigned rowids `42001` through `42004` to the rows above, those integers—not
-the descriptor strings—would be the Spark security keys.
-
-Conceptually, legacy allocation is:
-
-```text
-future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
-option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
-weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
-spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
-```
-
-#### Newer `IDGenerator`
-
-The namespace result is similar, but the third argument is chosen by the
-adapter. Examples using a description-oriented adapter are:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
-// store: ns="GLBX.FUTURE", key="ESZ6"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6 C6500"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
-
-get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
-// store: ns="COINBASE.SPOT", key="BTC-USD"
-```
-
-Databento intentionally produces a different logical row even when the human
-description is `ESZ6`. If its instrument ID were `12345678`, it would call the
-equivalent of:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
-// store: ns="GLBX.FUTURE", key="12345678"
-```
-
-Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
-`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
-while a provider that sends `exchange_security_id="BTCUSD"` produces a
-different row and therefore a different Spark key.
-
-These examples show why the tuple is only logical when the adapter's naming
-contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
-describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
-
-Both enum-prefix removals currently produce the same namespace spelling (for
-example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
-rowid space. Live rows cannot share a rowid. The table does not declare
-`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
-SQLite to reuse a number later. Security-key rows should therefore be treated
-as append-only operational data.
-
-`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
-and feed latency over maximum durability against a host/power failure. The
-unique `(ns, key)` index prevents two different rows for the same logical
-security.
-
-### Other places the key is stored
-
-| Location | Purpose | Authoritative for allocation? |
-|---|---|---|
-| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
-| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
-| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
-| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
-| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
-
-The `SecurityIDRecorder::work_db()` comment says a published security is also
-recorded in the security database, but that function does **not** write
-`securities.sqlite` in the current code. Full security definitions are persisted
-separately by `SecurityCache::flush_to_disk()` through
-`db::insert_or_replace()`.
-
-## CME MDP3 / OnixS pipeline in detail
-
-### 1. Construction and thread setup
-
-`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
-`MDProducer`. It constructs the recorder with:
-
-- a lookup callback that maps `ExchangeKey` through the CME index to the
-  handle's current Spark key; and
-- `Dispatcher::current()`, which drives a 200 ms recorder timer.
-
-In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
-dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
-then calls that thread's dispatcher `run()`. The older `md run onixs` command
-uses its command/dispatcher thread similarly.
-
-The OnixS feed engine has its own configured thread pool. Instrument and market
-data callbacks therefore originate on OnixS feed-engine threads, while recorder
-database work and final key assignment execute on the MDP3 dispatcher thread.
-
-| MDP3 stage | Executing thread | DB call? |
-|---|---|---|
-| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
-| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
-| Add CME-ID index entry | Same OnixS callback thread | No |
-| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
-| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
-| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
-
-### 2. Definition callback: exchange ID is set, Spark key is not
-
-For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
-
-```cpp
-fsec.numeric_id = def.securityId();
-```
-
-It also copies symbol/descriptor/venue and pricing fields. A newly constructed
-`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
-does not change it.
-
-For a spread, every leg starts with:
-
-```cpp
-leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
-leg.key = invalid_security_key;
-```
-
-Thus both the parent and legs initially carry exchange-routing identity, but no
-Spark identity.
-
-### 3. Shared-memory allocation and exchange indexing
-
-`MDP3Listener::add_security()` performs these operations in order:
-
-1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
-   cold entry and copies the security into the cold segment.
-2. Because the key is `0`, `MDProducer` queues the handle through
-   `SecurityIDRecorder::ensure_key()`.
-3. `_index.add_security(security.numeric_id, handle)` immediately creates the
-   CME-ID-to-handle mapping.
-4. The initial exchange status is written through the handle.
-
-This ordering means incoming books can already be routed by CME ID while the
-Spark key is pending. Shared-memory consumers cannot discover the security yet.
-Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
-when `key() != 0`, so a trade received in this short window updates the cold
-last-trade state but is not added to the keyed trade log.
-
-### 4. Recorder tick: spreads, handles, then database
-
-Every 200 ms the recorder calls, in this exact order:
-
-1. `work_spreads()` tries to fill previously unresolved spread leg keys by
-   asking `CMESecurityIndex` for each leg exchange ID.
-2. `work_handles()` drains new handles. Outrights are converted to an API
-   security and queued for DB lookup immediately. A spread without all leg keys
-   is retained in `pending_handle` and its leg pointers are registered in
-   `unresolved_legs`.
-3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
-
-Because spread work precedes new-handle work, a newly seen unresolved spread
-normally waits until at least the next timer tick to resolve legs. It waits
-longer if any outright leg has not yet received its Spark key. A spread's own
-key is not requested until every leg key is non-zero.
-
-### 5. Lookup or generation
-
-`work_db()` first queries `(ns, key)`:
-
-- Existing row: use its `rowid` as the key and invoke the callback.
-- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
-  rowid inside a transaction, the row object receives that ID, and the callback
-  runs after the transaction completes.
-
-New inserts are throttled to roughly 200 per timer pass (the current `> 200`
-check can admit 201) to avoid delaying heartbeats during startup. The queues are
-TBB concurrent queues, but all SQLite access is serialized onto the recorder's
-dispatcher thread. `check_access()` asserts this single-thread rule after the
-first work pass records the dispatcher thread ID.
-
-### 6. Final assignment and visibility
-
-The queued callback captures a copy of `MDProducerHandle` and calls
-`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
-then increments `security_count`. From this point:
-
-- shared-memory scans report the security;
-- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
-- trades and RFQs can be emitted with a non-zero key;
-- resolved spreads can copy this key into their leg structures.
-
-### 7. Sunday preload is a fast path
-
-For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
-definitions from `securities.sqlite`. It converts them with
-`set_security_key=true`, so both the security key and spread leg keys come from
-the serialized API security. `MDProducer::add_security()` calls `set_key()`
-immediately, and no recorder lookup is required for that preloaded handle. The
-CME exchange ID is still restored into `numeric_id`, allowing the normal CME
-index to be built.
-
-## Generalized/newer pipeline in detail
-
-### 1. Topology and ownership
-
-`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
-`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
-`MarketDataGateway`, and starts the configured providers. For each `(venue,
-source)` returned by a provider, the gateway creates an `MdSink` containing:
-
-- a venue-local `GeneralSecurityIndex`;
-- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
-- a `GeneralMarketDataListener` connecting the source to that producer/index.
-
-The source decides its callback thread. Some sources call the listener while
-starting; others own network or replay threads. The generalized listener calls
-the producer inline on that source thread. The process-wide `IDGenerator`
-serializes its own calls with a mutex, but the listener's transient
-`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
-expected to serialize security/event callbacks.
-
-| Generalized stage | Executing thread | DB call? |
-|---|---|---|
-| Build/parse domain definition | Source/adaptor thread | No |
-| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
-| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
-| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
-| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
-| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
-
-There are two important compatibility exceptions inside the provider framework:
-
-- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
-  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
-  legacy flow documented above.
-- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
-  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
-  producer/index. Its assignment is still synchronous and pre-populated, so it
-  resembles the newer ID flow even though it does not pass through
-  `GeneralMarketDataListener`.
-
-### 2. How a new ID is constructed
-
-For a normal live generalized adapter, ID creation happens **in the adapter,
-before** `GeneralMarketDataListener::on_securities()`:
-
-```text
-logical identity = (domain venue, domain security type, adapter-chosen name)
-Spark ID         = SQLite rowid for that identity in key_ids.sqlite
-```
-
-`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
-the enum prefixes:
-
-```text
-VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
-```
-
-It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
-row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
-next rowid. That integer is wrapped in `domain::SecurityID` and placed in
-`SecurityCommon::id`.
-
-The third identity component is adapter-specific, and this matters when trying
-to reproduce a mapping:
-
-| Source/adaptor family | Typical `name` sent to `IDGenerator` |
-|---|---|
-| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
-| Taifex | product/instrument ID string |
-| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
-| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
-| Emumba Deribit | extracted Emumba security descriptor |
-| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
-
-So “security name” alone is not a universal logical key. Reproduction requires
-the exact venue enum, exact security-type enum, and the exact adapter-specific
-name used at insertion time.
-
-### 3. What `GeneralMarketDataListener` does with that ID
-
-`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
-behavior is controlled by:
-
-```cpp
-const bool use_given_sec_key = true;
-```
-
-Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
-call the DB. It copies `security_common.id.value()` into its transient
-`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
-`_add_spot()` then calls `_copy_security_common()`, which sets:
-
-```cpp
-to.key = incoming_id;
-to.numeric_id = incoming_id;
-```
-
-This collapses the generalized source/correlation ID and Spark shared-memory
-key into the same number. The source's text `exchange_sec_id` remains on the
-domain security but is not copied into `FixedSecurity::numeric_id` by this
-path.
-
-`_commit_security()` then:
-
-1. calls `MDProducer::add_security()` with an already non-zero key;
-2. causes `MDProducerHandle::set_key()` to run immediately during that call;
-3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
-   number); and
-4. returns with the security already visible in shared memory.
-
-Because the sink's producer has no recorder, supplying `id == 0` would allocate
-the shared-memory entry but never assign or publish its key. Generalized live
-sources must therefore provide a valid non-zero ID before `on_securities()`.
-
-The listener contains an alternative branch for `use_given_sec_key == false`.
-That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
-sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
-also effectively unused. Turning the flag off is not a mechanical change: an
-adapter such as Databento allocates using numeric instrument ID while the
-listener branch would use `sec.desc()`, potentially creating a second key for
-the same instrument.
-
-There is also a spread failure edge case. `_find_sec(sec, true)` records the
-parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
-ID is absent, `_add_spread()` logs and returns without committing the spread. A
-later repeat of the parent definition is then classified as a duplicate and
-ignored, so definition ordering is a hard requirement in the current code.
-
-### 4. How later books, trades, and statistics find the security
-
-Every later domain event carries a `domain::SecurityID`:
-
-- `Book::security_id()`;
-- `TradeRecord::sec_id`; or
-- `SecurityMarketDataStatistics::security_id`.
-
-The listener first uses `cc_sec_id_map` to translate that event ID to the value
-stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
-`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
-but it still serves as the admission check that the definition was seen.
-
-Most high-throughput adapters avoid SQLite on market-data events:
-
-- Databento allocates on the definition and caches
-  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
-- Taifex keeps the resolved ID in its security-details mapper.
-- Other native subscriptions maintain comparable descriptor/ID maps.
-
-The Python bridge is a notable exception: `to_bts_security_()`,
-`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
-`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
-one SQLite `SELECT` for each converted Python definition, trade, or book. It
-returns the existing ID after the first insert, but it is still a DB call and
-takes the global `IDGenerator` mutex on every event.
-
-### 5. Persistence timing and latency
-
-The newer allocator is synchronous:
-
-- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
-  thread.
-- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
-  (SQLite autocommit), also on the caller's thread.
-- Listener publication: no further DB call under the current
-  `use_given_sec_key` setting.
-
-The ID therefore exists in `key_ids.sqlite` before the adapter sends the
-definition to the listener. There is no intentional 200 ms publication delay,
-but the source callback/startup path directly pays DB and mutex latency. Bulk
-definition discovery performs one lookup and possibly one insert per security;
-`IDGenerator` does not batch those operations.
-
-WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
-not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
-occasional file-system work. The impact is usually a startup/definition cost
-when adapters cache the result, but it becomes hot-path latency for code such as
-the current Python bridge that resolves every event through the DB.
-
-`GeneralMarketDataListener` itself does not persist the full contract definition
-to `securities.sqlite`; that remains the separate security-cache/metadata
-persistence path described earlier.
-
-## Comparison: MDP3 recorder vs generalized allocator
-
-| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
-|---|---|---|
-| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
-| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
-| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
-| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
-| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
-| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
-| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
-| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
-| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
-| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
-| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
-| Batching | Yes for new rows | No |
-| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
-
-Neither algorithm generates a UUID or a content hash. Both ultimately use an
-opaque, database-assigned SQLite rowid. It is normally increasing and looks like
-a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
-used to *look up* the row is logical; the resulting number is not derivable from
-that tuple.
-
-## Can another system reconstruct the same keys?
-
-Not from `(venue, type, name)` alone. Another system can reconstruct the
-**logical lookup tuple**, but it cannot calculate the numeric Spark key because
-the key depends on the historical insertion order of every row in
-`key_ids.sqlite`.
-
-For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
-row and receive different IDs if other securities were inserted before it on
-one system. They are sequential opaque IDs, not deterministic hashes and not
-UUIDs.
-
-Ways to preserve cross-system identity are:
-
-1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
-   identical. This preserves existing numeric keys but requires coordinated
-   allocation for new ones.
-2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
-   import rows with their original rowids before either system allocates new
-   securities.
-3. Treat a stable external identity (venue + type + exchange instrument ID, or
-   another canonical instrument identifier) as the cross-system key and carry a
-   translation table to each system's local Spark key.
-4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
-   be computed independently, but collision handling, descriptor
-   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
-   with orders/recordings/databases all need explicit design.
-
-Copying only `securities.sqlite` is insufficient as an allocator strategy. It
-contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
-the mapping used to find or allocate those IDs. Conversely, copying only
-`key_ids.sqlite` preserves the numbers but not all contract metadata.
-
-## Other assignment paths in `sparkd`
-
-The two main pipelines reduce to four ways a producer handle can acquire a key:
-
-1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
-   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
-   Sunday preload, relay playback, and other replay paths use this mechanism.
-2. **Recorder-backed live input.** The input key is zero and the producer owns a
-   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
-   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
-   `numeric_id` and otherwise follows the same recorder/index design.
-3. **Service assignment.** `SyntheticSecurityService` calls
-   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
-   callback receives a copied `api::Security` with its top-level `id` set, then
-   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
-   rowid allocator.
-4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
-   never becomes public. This is not a valid production assignment path, but it
-   is the failure mode if a generalized source omits its ID.
-
-Relay and replay deserve care when reasoning about persistence:
-
-- The generalized `MDReplayer` forwards the recorded domain security ID and
-  emits later books/statistics with the recorded `sec_key`, so the listener's
-  current use-given mode preserves the recording's numeric identity.
-- The legacy relay producer converts a recorded `api::Security` with
-  `set_security_key=true`, so its shared-memory publication also preserves the
-  embedded key.
-- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
-  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
-  separate mapping population step, not a deterministic derivation of the key.
-
-## Operational implications
-
-- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
-  definition continuity matter.
-- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
-  another host, even when all instruments are identical.
-- Before moving a feed between adapters, compare the exact `name` each adapter
-  supplies to `IDGenerator`. Matching venue/type is insufficient.
-- Cache generated IDs in the source after definition processing. Avoid calling
-  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
-  this guideline.
-- Keep generated rowids below the 32-bit shared-memory ceiling until the
-  `security_key` ABI is widened or explicit bounds checking is added to the
-  generalized path.
-- Ensure spread legs are announced before generalized spread parents.
-- Treat `synchronous=0` as a durability decision. A copied DB should be taken
-  with SQLite-aware backup/checkpoint handling rather than by casually copying
-  active WAL files.
-
-Useful inspection queries (run against the configured DB path) are:
-
-```sql
-SELECT rowid AS spark_key, ns, key
-FROM store
-ORDER BY rowid;
-
-SELECT rowid AS spark_key, ns, key
-FROM store
-WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
-```
-
-## Source map
-
-- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
-  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
-  and spread-leg resolution.
-- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
-  synchronous generalized lookup/allocation and process-wide mutex.
-- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
-  location, schema, WAL/synchronous pragmas, individual inserts, and batch
-  inserts.
-- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
-  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
-  shared-memory allocation, recorder handoff, and ordered publication.
-- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
-  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
-  widths and hot/cold storage fields.
-- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
-  and
-  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
-  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
-- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
-  producer/index insertion order.
-- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
-  current dedicated MDP3 dispatcher-thread ownership.
-- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
-  and
-  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
-  generalized source/sink topology, incoming-ID mapping, fixed-security
-  conversion, and publication.
-- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
-  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
-  and
-  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
-  representative adapter-side allocation and caching behavior.
-- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
-  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
-  persistence of full security definitions.
-- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
-  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
-  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
-  translation followed by Spark-keyed market-data consumption and MBO access.
-- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
-  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
-  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
-  representative downstream use of Spark security keys.
-- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
-  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
-  ID reuse.

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -1,44 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-RUN apt-get update && apt-get install -y tzdata
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet8
-
-RUN apt-get update && apt-get install -y gdb
-RUN apt-get update && apt-get install -y fish
-RUN apt-get update && apt-get install -y vim
-RUN apt-get update && apt-get install -y less
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts

docker/images/bluetradingsystems/linux_shell/Dockerfile

@@ -1,11 +0,0 @@
-FROM ubuntu:24.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y \
-  fish \
-  less \
-  vim
-
-USER bts
-WORKDIR /home/bts

docker/ubuntu-22.04/dotnet-7/Dockerfile

@@ -0,0 +1,39 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+  && apt-get install -y pkg-config \
+  && apt-get install -y locales \
+  && apt-get install -y ca-certificates \
+  && apt-get install -y libssh2-1-dev \
+  && apt-get install -y software-properties-common \
+  && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

requirements.txt

@@ -18,10 +18,10 @@ MarkupSafe==3.0.2
 mccabe==0.6.1
 numpy==2.2.6
 platformdirs==2.3.0
-protobuf==6.33.4
+protobuf==4.21.12
 pybars3==0.9.7
-pybind11==3.0.1
-pybind11-stubgen==2.5.5
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
 pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20

spark_client.sln

@@ -1,306 +1,164 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|Any CPU = Release|Any CPU
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-	EndGlobalSection
-EndGlobal
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		MinSizeRel|x64 = MinSizeRel|x64
+		Release|x64 = Release|x64
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
+	EndGlobalSection
+EndGlobal

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -11,20 +11,10 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ArithmeticExpression">
     <Scanner Include="ArithmeticExpressionTokens.lex" />

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -5,5 +5,9 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
-  
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
 </Project>

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,80 +6,23 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var obj1 = value1.GetValue(env);
-      var obj2 = value2.GetValue(env);
-      if (obj1 is null) {
-         if (obj2 is null) {
-            return 0;
-         }
-         return -1;
-      }
-      if (obj2 is null) {
-         return 1;
-      }
-      return compareObjs(obj1, obj2);
-   }
-
-   static int compareObjs(object obj1, object obj2) {
+      var cmp1 = value1.Cast<IComparable>();
+      var cmp2 = value2.Cast<IComparable>();
       try {
-         var typ1 = obj1.GetType();
-         var typ2 = obj2.GetType();
-         bool areSameTypes = typ1 == typ2;
-         if (areSameTypes) {
-            return ((IComparable)obj1).CompareTo(obj2);
-         }
-         if (typ1.IsNumeric()) {
-            return tryCompareNumeric(obj1, obj2);
-         } else if (typ2.IsNumeric()) {
-            return -tryCompareNumeric(obj2, obj1);
-         } else if (typ1.IsEnum) {
-            if (typ2 == typeof(string)) {
-               return -((string)obj2).CompareTo(obj1.ToString());
-            }
-         } else if (typ1 == typeof(string)) {
-            return ((string)obj1).CompareTo(obj2.ToString());
-         } else if (typ1 == typeof(DateTime)) {
-            if (typ2 == typeof(TimeSpan)) {
-               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
-            }
-         } else if (typ1 == typeof(TimeSpan)) {
-            if (typ2 == typeof(DateTime)) {
-               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
-            }
-         }
+         var v1 = cmp1.GetValue(env);
+         var v2 = cmp2.GetValue(env);
+         return v1.GetType().IsNumeric() ?
+            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
-      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
-   }
-
-   static int tryCompareNumeric(object numericObj1, object obj2) {
-      var typ2 = obj2.GetType();
-      if (typ2.IsNumeric()) {
-         return new NumericComparer().Compare(numericObj1, obj2);
-      } else if (typ2.IsEnum) {
-         var undTyp2 = typ2.GetEnumUnderlyingType();
-         if (undTyp2 == typeof(byte)) {
-            return new NumericComparer().Compare(numericObj1, (byte)obj2);
-         }
-         if (undTyp2 == typeof(short)) {
-            return new NumericComparer().Compare(numericObj1, (short)obj2);
-         }
-         if (undTyp2 == typeof(int)) {
-            return new NumericComparer().Compare(numericObj1, (int)obj2);
-         }
-         if (undTyp2 == typeof(long)) {
-            return new NumericComparer().Compare(numericObj1, (long)obj2);
-         }
-      }
-      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -41,7 +41,6 @@ Identifier               {Letter}({Letter}*[0-9]*_*)*
 OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
-Negate                   '!'
 MapsTo                   ':'
 IsOfType                 '/'
 
@@ -94,10 +93,6 @@ IsOfType                 '/'
    return (int)Token.COMMA;
 }
 
-{Negate} {
-   return (int)Token.NEGATE;
-}
-
 {MapsTo} {
    return (int)Token.MAPS_TO;
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -11,20 +11,10 @@
     <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
   
   <ItemGroup>
     <Scanner Include="BooleanExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -10,20 +10,11 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnExpression">
     <Scanner Include="ColumnExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -10,20 +10,11 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnSortExpression">
     <Scanner Include="ColumnSortExpressionTokens.lex" />

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.development

@@ -0,0 +1 @@
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.production

@@ -0,0 +1 @@
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>true</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/ico" href="/logo.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>BTS Cloud Streams</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
+    "classnames": "^2.5.1",
+    "react": "^18.3.1",
+    "react-data-grid": "^7.0.0-beta.44",
+    "react-data-table-component": "^7.6.2",
+    "react-dom": "^18.3.1",
+    "react-json-view-lite": "^1.4.0",
+    "react-pro-sidebar": "^1.1.0",
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "react-mui-sidebar": "^1.3.5",
+    "vite": "^5.2.13"
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/AnimatedSideBar.jsx

@@ -0,0 +1,40 @@
+import { Sidebar, Menu, MenuItem, Submenu, Logo } from "react-mui-sidebar";
+import './App.css';
+
+const AnimatedSideBar = () => {
+  return (
+    <Sidebar className={"sidebar-container"} width={"270px"}>
+      <Logo img="https://adminmart.com/wp-content/uploads/2024/03/logo-admin-mart-news.png">
+        AdminMart
+      </Logo>
+      <Menu subHeading="HOME">
+        <MenuItem link="/" badge="true">
+          Modern
+        </MenuItem>
+        <MenuItem>eCommerce</MenuItem>
+        <MenuItem>Analytical</MenuItem>
+      </Menu>
+      <Menu subHeading="APPS">
+        <MenuItem>Chat</MenuItem>
+        <MenuItem>Calendar</MenuItem>
+      </Menu>
+      <Menu subHeading="OTHERS">
+        <Submenu title="Menu Level">
+          <MenuItem>Post</MenuItem>
+          <MenuItem>Details</MenuItem>
+          <Submenu title="Level 2">
+            <MenuItem>new</MenuItem>
+            <MenuItem>Hello</MenuItem>
+          </Submenu>
+        </Submenu>
+
+        <MenuItem>Chip</MenuItem>
+        <MenuItem target="_blank" link="google.com">
+          External Link
+        </MenuItem>
+      </Menu>
+    </Sidebar>
+  );
+};
+
+export default AnimatedSideBar;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -0,0 +1,101 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+.root-div {
+  display: flex;
+  flex-direction: row;
+  align-items: flex-start;
+}
+
+.main-content-div {
+  display: flex;
+  flex-direction: row;
+}
+
+.title-div {
+  width: 100%;
+}
+
+.container {
+  display: flex;
+}
+
+.container-column {
+  display: flex;
+  flex-direction: column;
+}
+
+.container-row {
+  display: flex;
+  flex-direction: row;
+  align-items: stretch;
+}
+
+box.query-input-form {
+   /* p: 2, display: "flex", flexFlow: "column", alignItems: "stretch"  */
+  display: flex;
+  flex-flow: column;
+  align-items: stretch;
+}
+
+/* li.ps-menuitem-root:hover {
+  background-color: #ff9922;  
+} */
+
+.container-row-centered {
+  display: flex;
+  flex-direction: row;
+  justify-content: center;
+  align-items: center;
+}
+
+h1#main-header {
+  padding: 4px 8px 4px 8px;
+  text-align: center;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
+
+/* Sidebar.css */
+/* .sidebar-container {
+  display: flex;
+  flex-direction: column;
+}
+
+.sidebar {
+  position: relative;
+  top: 0;
+  left: 0;
+  height: 100vh;
+  background-color: #333;
+  color: white;
+  overflow: hidden;
+  transition: transform 0.3s ease;
+}
+
+.sidebar.closed {
+  width: 0;
+}
+
+.sidebar.open {
+  width: 250px;
+} */

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -0,0 +1,173 @@
+import { ThemeProvider, CssBaseline, GlobalStyles } from '@mui/material';
+import QueryStatusView from './QueryStatusView';
+import useMediaQuery from '@mui/material/useMediaQuery';
+import QueryControl from './QueryControl';
+import DataTable from './DataTable';
+import { useState, useEffect, useMemo } from 'react';
+import useWebSocket, { ReadyState } from 'react-use-websocket';
+import Sidebar from './SideBar';
+import getTheme from './Theme';
+
+import '@fontsource/noto-sans';
+import './App.css';
+import './SideBar.css';
+
+const webSocketUrl = import.meta.env.VITE_DATASTREAM_WEBSOCKET_URL;
+// Always use the same subscription id (1), because we only show 1 table at a time.
+// The server will automatically dispose the previous subscription and start a new one
+// if we're currently subscribed to another query.
+const subscriptionId = "1";
+
+var rpcReqIdCounter = 1;
+
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: subscriptionId,
+      query: query
+    }
+  };
+}
+
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: subscriptionId,
+    }
+  };
+}
+
+const App = () => {
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
+  const globalStyles = {
+    ':root': {
+      '--primary-color': theme.palette.primary.main,
+      '--primary-dark-color': theme.palette.primary.dark,
+      '--secondary-color': theme.palette.secondary.main,
+      '--secondary-dark-color': theme.palette.secondary.dark,
+      '--background-color': theme.palette.background.default,
+    },
+  };
+  const [blueTable, setBlueTable] = useState({ columns: null, rows: null });
+  const [queryError, setQueryError] = useState(null);
+  const [queryState, setQueryState] = useState("Unsubscribed");
+  const queryStateDetails = useMemo(() => {
+    if (queryState == "Unsubscribed") {
+      return "Enter query above, then click the play button to subscribe";
+    }
+    if (queryState == "Subscribed") {
+      return null;
+    }
+  }, [queryState]);
+  const [sidebarIsOpen, setSidebarIsOpen] = useState(false);
+  const [cannedQuery, setCannedQuery] = useState(null);
+
+  const onCannedQueryClicked = (cannedQuery) => {
+    console.info(`Executing canned query: ${JSON.stringify(cannedQuery)}`);
+    const queryText = cannedQuery.query.join("\n");
+    setCannedQuery({ queryText: queryText, clickTime: Date.now() });
+    subscribe(queryText);
+  };
+
+  const { sendJsonMessage, getWebSocket, readyState } = useWebSocket(
+    webSocketUrl,
+    {
+      onOpen: (e) => console.info(`WebSocket open. ${JSON.stringify(e)}`),
+      onClose: (e) => console.info(`WebSocket closed. ${JSON.stringify(e)}`),
+      onError: (e) => console.error(`WebSocket error: ${JSON.stringify(e)}`),
+      shouldReconnect: () => true,
+    });
+
+  useEffect(() => {
+    console.info(`WebSocket readyState: ${readyState}`)
+    if (readyState != ReadyState.OPEN) {
+      return;
+    }
+    const ws = getWebSocket();
+    ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      setQueryError(msg.error);
+      if (msg.error != null) {
+        setQueryState("Error");
+      } else if (msg.id != null) {
+        setQueryState(msg.result.status);
+      } else {
+        console.assert(msg.params.subscription_id == subscriptionId);
+        console.info(`Received rows: ${msg.params.table.all}`);
+        setBlueTable({ columns: msg.params.table.columns, rows: msg.params.table.all });
+      }
+    };
+  }, [getWebSocket, readyState]);
+
+  const subscribe = (query) => {
+    const req = subscribeReq(query);
+    console.info(`Subscribing: ${JSON.stringify(subscribeReq(query))}`);
+    sendJsonMessage(req);
+  };
+
+  const unsubscribe = () => {
+    const req = unsubscribeReq();
+    console.info(`Unsubscribing: ${JSON.stringify(req)}`);
+    sendJsonMessage(req);
+  };
+
+  const onOpened = () => {
+    setSidebarIsOpen(true);
+  };
+
+  const onClosed = () => {
+    setSidebarIsOpen(false);
+  };
+
+  const sidebarWidth = sidebarIsOpen ? '250px' : '65px';
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <GlobalStyles styles={globalStyles} />
+      <div className='root-div'>
+        <div style={{
+          display: 'flex',
+          flexDirection: 'column',
+          alignItems: 'flex-start',
+          flexWrap: 'nowrap',
+          width: sidebarWidth,
+          transition: 'width 0.3s ease',
+        }}>
+          <Sidebar onSelection={onCannedQueryClicked} onOpened={onOpened} onClosed={onClosed} />
+        </div>
+        <div style={{
+          display: 'flex',
+          flexDirection: 'column',
+          left: sidebarWidth,
+          width: '100%',
+          transition: 'width 0.3s ease',
+        }}>
+          <div style={{ width: '100%', display: 'flex', flexDirection: 'column', alignItems: 'flex-start', margin: theme.spacing(2, 0, 0, 0) }}>
+            <div style={{ width: '90%', rightMargin: '100px' }}>
+              <QueryControl
+                queryState={queryState}
+                queryText={cannedQuery?.queryText}
+                queryTextResetTime={cannedQuery?.clickTime}
+                subscribe={subscribe}
+                unsubscribe={unsubscribe} />
+            </div>
+            <div style={{ width: '90%', rightMargin: '100px' }}>
+              <QueryStatusView queryState={queryState} details={queryStateDetails} error={queryError} />
+              <DataTable columns={blueTable?.columns ?? []} rows={blueTable?.rows ?? []} />
+            </div>
+          </div>
+        </div>
+      </div>
+    </ThemeProvider>
+  );
+};
+
+export default App;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataGrid.css

@@ -0,0 +1,53 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+.data-grid-container {
+  overflow: auto;
+  white-space: nowrap;
+}
+
+
+.data-grid {
+  max-height: 1000px;
+}
+
+div[role=columnheader] {
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+}
+
+/* div[role=columnheader] {
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+}
+
+.rdg-row {
+  border: 0px;
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: rgba(var(--primary-dark-color), 0.8);
+  color: var(--primary-color);
+}
+
+.rdg-row-odd {
+  border: 0px;
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+  color: var(--primary-color);
+}
+
+.rdg-cell {
+  border-bottom: 0px;
+}
+
+.rdg-cell.even {
+  border-bottom: 5px;
+}
+
+.rdg-cell.odd {
+  border-bottom: 5px;
+} */

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.css

@@ -0,0 +1,32 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+th {
+  font-size: large;  
+  padding: 15px;
+}
+
+td {
+  padding: 15px;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.jsx

@@ -0,0 +1,74 @@
+import PropTypes from 'prop-types';
+import { useTheme } from '@mui/material/styles';
+import { DataGrid } from '@mui/x-data-grid';
+import './DataGrid.css';
+import { useMemo, useState } from 'react';
+
+export default function DataTable({ columns, rows }) {
+  const theme = useTheme();
+  const [columnWidths, setColumnWidths] = useState({});
+
+  const handleColumnResize = (params) => {
+    const newWidth = params.width;
+    setColumnWidths((prevWidths) => ({
+      ...prevWidths,
+      [params.colDef.field]: params.width,
+    }));
+  };
+
+  const getGridColDef = useMemo(() => {
+    return col => {
+      return {
+        field: col.name,
+        headerName: col.name,
+        width: columnWidths[col.name],
+      };
+    };
+  }, [columnWidths]);
+
+  return (
+    <DataGrid
+      className='data-grid-container'
+      autosizeOnMount={false}
+      columns={columns?.map(getGridColDef) ?? []}
+      onColumnResize={handleColumnResize}
+      rows={rows ?? []}
+      sx={{
+        margin: theme.spacing(2, 0, 0, 0),
+        borderColor: 'primary.main',
+        backgroundColor: 'primary.dark',
+        color: 'text.primary',
+        '& .MuiDataGrid-columnHeader': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-filler': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-cell': {
+          color: 'text.primary',
+        },
+        '& .MuiDataGrid-cell:hover': {
+          color: 'primary.main',
+        },
+        '& .MuiDataGrid-row': {
+          '--DataGrid-rowBorderColor': 'secondary.dark'
+        }
+      }}
+    />
+  );
+}
+
+DataTable.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+  apiRef: PropTypes.object,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryControl.jsx

@@ -0,0 +1,119 @@
+import PropTypes from 'prop-types';
+import DataTable from './DataTable';
+import { useRef, useState, useEffect } from 'react';
+import { useGridApiRef } from '@mui/x-data-grid';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';
+
+const QueryControl = (props) => {
+  const { queryState, queryText, queryTextResetTime, subscribe, unsubscribe } = props;
+  // const theme = useTheme();
+  let textInput = useRef();
+  const [queryTextChangeTime, setQueryTextChangeTime] = useState(null);
+  useEffect(() => {
+    if (queryText == null || textInput.current == null) {
+      return;
+    }
+    // Init text field (unless a non-empty string is already present, or the user has modified
+    // the query - we don't want to overwrite their changes).
+    if (queryTextChangeTime == null || queryTextResetTime > queryTextChangeTime) {
+      console.info(`Initializing query text: ${queryText}`)
+      textInput.current.value = queryText;
+    }
+  }, [queryText, queryTextChangeTime, queryTextResetTime]);
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };
+
+  const onRunClick = () => {
+    subscribe(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    unsubscribe();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
+    }
+  };
+
+  const onChange = (e) => {
+    // So we know not to overwrite the changes with the canned query on re-render
+    setQueryTextChangeTime(Date.now());
+  };
+
+  return (
+    <Box sx={{ display: "flex", flexFlow: "column" }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onChange={onChange}
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `queryStartButton.${queryState}.color`,
+                  bgcolor: `queryStartButton.${queryState}.backgroundColor`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `queryStopButton.${queryState}.color`,
+                  bgcolor: `queryStopButton.${queryState}.backgroundColor`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};
+
+QueryControl.propTypes = {
+  columns: PropTypes.array,
+  rows: PropTypes.array,
+  isRunning: PropTypes.bool,
+  queryText: PropTypes.string,
+  queryTextResetTime: PropTypes.number,
+  queryState: PropTypes.string,
+  subscribe: PropTypes.func,
+  unsubscribe: PropTypes.func,
+};
+
+export default QueryControl;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryStatusView.jsx

@@ -0,0 +1,73 @@
+import { useState } from 'react';
+import PropTypes from 'prop-types';
+import { Box, Typography, Collapse } from '@mui/material';
+import Alert from '@mui/material/Alert';
+import AlertTitle from '@mui/material/AlertTitle';
+import { useTheme } from '@mui/material/styles';
+import { alpha } from '@mui/system';
+
+const QueryStatusView = ({ queryState, details, error }) => {
+  const theme = useTheme();
+  const [open, setOpen] = useState(false);
+
+  let header = queryState;
+  let headerContent = null;
+  let detailElements = [];
+  let severity = queryState == "Subscribed" ? "success" : "info";
+  let cursor = details == null ? "default" : "pointer";
+  if (error != null) {
+    severity = "error";
+    cursor = "pointer";
+    header = `Error Code: ${error.code}`;
+    headerContent = error.message;
+    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
+    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
+    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
+      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
+        {line}
+      </Typography>));
+  } else if (details != null) {
+    detailElements.push(<Typography variant="body1">{details}</Typography>);
+  }
+  return (
+    <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
+      <Alert
+        severity={severity}
+        onClick={() => setOpen(!open)}
+        sx={{
+          margin: theme.spacing(2, 0, 0, 0),
+          cursor: cursor,
+        }}>
+        <AlertTitle>{header}</AlertTitle>
+        {headerContent}
+      </Alert>
+      {detailElements.length == 0
+        ? null
+        : <Collapse in={open}>
+          <Box
+            sx={{
+              backgroundColor: alpha(theme.palette[severity].light, 0.75),
+              color: theme.palette[severity].contrastText,
+              padding: theme.spacing(2),
+              marginTop: theme.spacing(0),
+              borderRadius: theme.shape.borderRadius,
+            }}
+          >
+            {detailElements}
+          </Box>
+        </Collapse>}
+    </Box>
+  );
+};
+
+QueryStatusView.propTypes = {
+  queryState: PropTypes.string,
+  details: PropTypes.string,
+  error: PropTypes.shape({
+    code: PropTypes.number,
+    message: PropTypes.string,
+    data: PropTypes.object,
+  })
+};
+
+export default QueryStatusView;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/ReactDataGrid.jsx

@@ -0,0 +1,27 @@
+import 'react-data-grid/lib/styles.css';
+import PropTypes from 'prop-types';
+import DataGrid from 'react-data-grid';
+import './DataGrid.css';
+
+function getGridColDef(col) {
+  return {
+    key: col.name,
+    name: col.name,
+  };
+}
+
+export default function ReactDataGrid({columns, rows}) {
+  return (
+    <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
+  );
+}
+
+ReactDataGrid.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.css

@@ -0,0 +1,98 @@
+/* Sidebar.css */
+.app-container {
+  display: flex;
+  width: 100%;
+  overflow: hidden;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: flex-start;
+}
+
+.sidebar {
+  display: flex;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: flex-start;
+  background-color: transparent;
+  color: white;
+  overflow: hidden;
+  transition: width 0.3s ease;
+}
+
+.sidebar.open {
+  width: 100%;
+}
+
+.sidebar.closed {
+  width: 0%;
+}
+
+.sidebar-toggle-button {
+  border: 0px;
+  border-radius: 50%;
+  box-shadow: rgba(9, 30, 66, 0.08) 0px 0px 0px 1px, rgba(9, 30, 66, 0.08) 0px 2px 4px 1px;
+  color: var(--primary-color);
+  cursor: pointer;
+  height: 24px;
+  opacity: 1;
+  outline: 0px;
+  padding: 0px;
+  background-color: transparent;
+  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
+  width: 20%;
+}
+
+.sidebar-toggle-button:hover {
+  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
+}
+
+.main-content.open {
+  width: 165px;
+  transition: width 0.3s ease;
+  background-color: transparent;
+}
+
+.main-content.closed {
+  width: 0%;
+  /* min-width: 25px; */
+  transition: width 0.3s ease;
+  background-color: transparent;
+}
+
+.sidebar-menu-item {
+  width: 100%;
+  box-sizing: border-box;
+}
+
+.sidebar-menu-item.open {
+  width: 100%;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item.closed {
+  width: 0%;
+  min-width: 0px;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item-content.open {
+  width: 100%;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item-content.closed {
+  width: 0px;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+/* Ensure span can shrink */
+.button-text {
+  display: block;    /* Make span a block element */
+  width: 100%;       /* Ensure it takes the full width of its parent */
+  overflow: hidden;  /* Hide any overflow content */
+  white-space: nowrap; /* Prevent text wrapping */
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.jsx

@@ -0,0 +1,109 @@
+import { useState, useEffect } from 'react';
+import Button from '@mui/material/Button';
+import { useTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import Tooltip from '@mui/material/Tooltip';
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded";
+import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
+import FileCopyIcon from '@mui/icons-material/FileCopy';
+import './SideBar.css';
+
+const Sidebar = ({ onSelection, onOpened, onClosed }) => {
+  const theme = useTheme();
+  const [isOpen, setIsOpen] = useState(false);
+  const [cannedQueries, setCannedQueries] = useState([]);
+  const Icon = isOpen ? ChevronLeftRounded : ChevronRightRounded;
+
+  const toggleIsOpen = () => {
+    if (isOpen) {
+      setIsOpen(false);
+      if (onClosed != null) {
+        onClosed();
+      }
+    } else {
+      setIsOpen(true);
+      if (onOpened != null) {
+        onOpened();
+      }
+    }
+  };
+
+  const onMenuItemClick = (event, cannedQuery) => {
+    console.info(`menu item clicked: ${cannedQuery.name}`);
+    onSelection(cannedQuery);
+  };
+
+  // const onMenuItemClick = async (event, cannedQuery) => {
+  //   try {
+  //     await navigator.clipboard.writeText(cannedQuery.query.join("\n"));
+  //     alert(`Copied ${cannedQuery.name} to clipboard`);
+  //   } catch (err) {
+  //     console.error('Failed to copy query text: ', err);
+  //   }
+  // };
+
+  useEffect(() => {
+    const fetchCannedQueries = async () => {
+      try {
+        const response = await fetch('api/cannedqueries');
+        const data = await response.json();
+        setCannedQueries(data);
+      } catch (error) {
+        console.error('Error fetching WebSocket config:', error);
+      }
+    };
+    fetchCannedQueries();
+  }, []);
+
+  const mainContentClass = `main-content ${isOpen ? 'open' : 'closed'}`;
+  const menuItemClass = `sidebar-menu-item ${isOpen ? 'open' : 'closed'}`;
+  const menuItemContentClass = `sidebar-menu-item-content ${isOpen ? 'open' : 'closed'}`;
+  const toggleButtonTitle = `${isOpen ? 'Hide' : 'Show'} Common Queries`;
+
+  return (
+    <div
+      className={"sidebar"}
+      style={{
+        display: 'flex',
+        flexDirection: 'row',
+        justifyContent: 'flexStart',
+        alignItems: 'flexStart',
+        margin: theme.spacing(2, 0, 0, 0)
+      }}
+    >
+      <div className={mainContentClass}>
+        {cannedQueries.map(cq =>
+          <Button key={cq.name} className={menuItemClass} onClick={(event) => onMenuItemClick(event, cq)}>
+            <div className={menuItemContentClass}>
+              <span className='button-text'>{cq.name}</span>
+            </div>
+          </Button>)
+        }
+        {/* {cannedQueries.map(cq =>
+          <Button
+            variant="contained"
+            color="primary"
+            startIcon={<FileCopyIcon />}
+            onClick={onMenuItemClick}
+          >
+            <div className={menuItemContentClass}>
+              <span className='button-text'>{cq.name}</span>
+            </div>
+          </Button>)} */}
+      </div>
+      <Tooltip title={toggleButtonTitle}>
+        <Button onClick={toggleIsOpen}>
+          <Icon className="w-5 h-5" />
+        </Button>
+      </Tooltip>
+    </div>
+  );
+};
+
+export default Sidebar;
+
+Sidebar.propTypes = {
+  onSelection: PropTypes.func,
+  onOpened: PropTypes.func,
+  onClosed: PropTypes.func,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/Theme.js

@@ -0,0 +1,112 @@
+import { createTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const getLightPalette = () => {
+  return {
+    mode: 'light',
+    primary: {
+      light: '#6573c3',
+      main: '#3f51b5',
+      dark: '#2c387e',
+      contrastText: '#ffffff',
+    },
+    secondary: {
+      light: '#f73378',
+      main: '#f50057',
+      dark: '#ab003c',
+      contrastText: '#ffffff',
+    },
+    text: {
+      primary: '#000000c4',
+      secondary: '#00000086',
+      disabled: '#0000005b',
+      hint: '#0000005b',
+    }
+  };
+};
+
+const getDarkPalette = () => {
+  return {
+    mode: 'dark',
+    primary: {
+      light: '#bcc5ff',
+      main: '#33baf9',
+      dark: '#202334',
+      contrastText: '#78a2ac',
+    },
+    secondary: {
+      light: '#9fd0b8',
+      main: '#5eb289',
+      dark: '#ffffff12',
+      contrastText: '#c8d4f7',
+    },
+    action: {
+      hoverOpacity: 0.08
+    },
+    text: {
+      primary: '#eaf8f8',
+      dark: '#eaf8f8a0'
+    },
+    queryStartButton: {
+      Subscribed: {
+        color: '#33baf9',
+        backgroundColor: '#ffffff40',
+      },
+      Unsubscribed: {
+        color: '#ffffff40',
+        backgroundColor: '#ffffff00',
+      }
+    },
+    queryStopButton: {
+      Subscribed: {
+        color: '#ff4444a0',
+        backgroundColor: '#ffffff40',
+      },
+      Unsubscribed: {
+        color: '#ff444450',
+        backgroundColor: '#ffffff00',
+      }
+    },
+    error: {
+      light: '#e57373',
+      main: '#f44336',
+      dark: '#d32f2f',
+      contrastText: '#fff',
+    },
+    warning: {
+      light: '#ffb74d',
+      main: '#ff9800',
+      dark: '#f57c00',
+      contrastText: '#000',
+    },
+    info: {
+      light: '#64b5f6',
+      main: '#2196f3',
+      dark: '#303030',
+      contrastText: '#fff',
+    },
+    success: {
+      light: '#81c784',
+      main: '#4caf50',
+      dark: '#388e3c',
+      contrastText: '#000',
+    },
+  };
+};
+
+const getTheme = ({ mode }) => {
+  return createTheme({
+    typography: {
+      fontFamily: '"Helvetica"'
+    },
+    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
+  });
+};
+
+export default getTheme;
+
+getTheme.propTypes = {
+  mode: PropTypes.string.isRequired,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark-hc.css

@@ -0,0 +1,51 @@
+.dark-high-contrast {
+  --md-sys-color-primary: rgb(251 250 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 0 0);
+  --md-sys-color-primary-container: rgb(173 204 255);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(250 250 255);
+  --md-sys-color-on-secondary: rgb(0 0 0);
+  --md-sys-color-secondary-container: rgb(159 208 255);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(255 249 251);
+  --md-sys-color-on-tertiary: rgb(0 0 0);
+  --md-sys-color-tertiary-container: rgb(222 193 231);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 249 249);
+  --md-sys-color-on-error: rgb(0 0 0);
+  --md-sys-color-error-container: rgb(255 186 177);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(255 255 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(251 250 255);
+  --md-sys-color-outline: rgb(200 202 212);
+  --md-sys-color-outline-variant: rgb(200 202 212);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(0 0 0);
+  --md-sys-color-inverse-primary: rgb(0 42 83);
+  --md-sys-color-primary-fixed: rgb(219 231 255);
+  --md-sys-color-on-primary-fixed: rgb(0 0 0);
+  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
+  --md-sys-color-secondary-fixed: rgb(214 233 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
+  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
+  --md-sys-color-tertiary-fixed: rgb(249 222 255);
+  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
+  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark-mc.css

@@ -0,0 +1,51 @@
+.dark-medium-contrast {
+  --md-sys-color-primary: rgb(173 204 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 22 49);
+  --md-sys-color-primary-container: rgb(112 146 198);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(159 208 255);
+  --md-sys-color-on-secondary: rgb(0 24 42);
+  --md-sys-color-secondary-container: rgb(100 149 193);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(222 193 231);
+  --md-sys-color-on-tertiary: rgb(33 14 42);
+  --md-sys-color-tertiary-container: rgb(162 136 171);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 186 177);
+  --md-sys-color-on-error: rgb(55 0 1);
+  --md-sys-color-error-container: rgb(255 84 73);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(251 250 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(200 202 212);
+  --md-sys-color-outline: rgb(160 163 171);
+  --md-sys-color-outline-variant: rgb(128 131 139);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(40 42 47);
+  --md-sys-color-inverse-primary: rgb(35 73 120);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 17 40);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark.css

@@ -0,0 +1,51 @@
+.dark {
+  --md-sys-color-primary: rgb(165 200 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 49 94);
+  --md-sys-color-primary-container: rgb(34 72 118);
+  --md-sys-color-on-primary-container: rgb(212 227 255);
+  --md-sys-color-secondary: rgb(154 203 250);
+  --md-sys-color-on-secondary: rgb(0 51 83);
+  --md-sys-color-secondary-container: rgb(12 74 114);
+  --md-sys-color-on-secondary-container: rgb(206 229 255);
+  --md-sys-color-tertiary: rgb(218 189 226);
+  --md-sys-color-on-tertiary: rgb(61 41 70);
+  --md-sys-color-tertiary-container: rgb(85 63 93);
+  --md-sys-color-on-tertiary-container: rgb(247 216 255);
+  --md-sys-color-error: rgb(255 180 171);
+  --md-sys-color-on-error: rgb(105 0 5);
+  --md-sys-color-error-container: rgb(147 0 10);
+  --md-sys-color-on-error-container: rgb(255 218 214);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(225 226 233);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(195 198 207);
+  --md-sys-color-outline: rgb(141 145 153);
+  --md-sys-color-outline-variant: rgb(67 71 78);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(46 48 53);
+  --md-sys-color-inverse-primary: rgb(60 96 144);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light-hc.css

@@ -0,0 +1,51 @@
+.light-high-contrast {
+  --md-sys-color-primary: rgb(0 34 70);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(29 68 114);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(0 36 61);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(3 70 110);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(46 26 55);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(81 59 89);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(78 0 2);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(140 0 9);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(0 0 0);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(32 36 43);
+  --md-sys-color-outline: rgb(63 67 74);
+  --md-sys-color-outline-variant: rgb(63 67 74);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(255 255 255);
+  --md-sys-color-inverse-primary: rgb(228 236 255);
+  --md-sys-color-primary-fixed: rgb(29 68 114);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(3 70 110);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(81 59 89);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light-mc.css

@@ -0,0 +1,51 @@
+.light-medium-contrast {
+  --md-sys-color-primary: rgb(29 68 114);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(83 118 167);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(3 70 110);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(71 121 163);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(81 59 89);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(133 108 141);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(140 0 9);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(218 52 46);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(63 67 74);
+  --md-sys-color-outline: rgb(91 95 103);
+  --md-sys-color-outline-variant: rgb(119 123 131);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(83 118 167);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(71 121 163);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(133 108 141);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light.css

@@ -0,0 +1,51 @@
+.light {
+  --md-sys-color-primary: rgb(60 96 144);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(212 227 255);
+  --md-sys-color-on-primary-container: rgb(0 28 58);
+  --md-sys-color-secondary: rgb(46 98 140);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(206 229 255);
+  --md-sys-color-on-secondary-container: rgb(0 29 50);
+  --md-sys-color-tertiary: rgb(110 86 118);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(247 216 255);
+  --md-sys-color-on-tertiary-container: rgb(39 20 48);
+  --md-sys-color-error: rgb(186 26 26);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(255 218 214);
+  --md-sys-color-on-error-container: rgb(65 0 2);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(67 71 78);
+  --md-sys-color-outline: rgb(116 119 127);
+  --md-sys-color-outline-variant: rgb(195 198 207);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -0,0 +1,107 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+button#play-button {
+  padding: '10px 20px';
+  font-size: '16px';
+  cursor: 'pointer';
+  background-color: '#28a745';
+  color: '#fff';
+  border: 'none';
+  border-radius: '5px';
+};
+
+button#pause-button {
+  background-color: #ffc107;
+}
+
+button#stop-button {
+  background-color: #dc3545;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -0,0 +1,7 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <App />
+)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -0,0 +1,80 @@
+import { fileURLToPath, URL } from 'node:url';
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+import process, { env } from 'process';
+
+const baseFolder =
+  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+    ? `${process.env.APPDATA}/ASP.NET/https`
+    : `${process.env.HOME}/.aspnet/https`;
+
+const certificateName = "bts.datastreamwebapp.client";
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+  if (0 !== child_process.spawnSync('dotnet', [
+    'dev-certs',
+    'https',
+    '--export-path',
+    certFilePath,
+    '--format',
+    'Pem',
+    '--no-password',
+  ], { stdio: 'inherit', }).status) {
+    throw new Error("Could not create certificate.");
+  }
+}
+
+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:8443';
+
+export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  return {
+    plugins: [plugin()],
+    resolve: {
+      alias: {
+        '@': fileURLToPath(new URL('./src', import.meta.url))
+      }
+    },
+    build: {
+      minify: false,
+      sourcemap: true,
+      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
+    },
+    server: {
+      proxy: {
+        '^/api': {
+          target,
+          secure: false,
+          configure: (proxy) => {
+            proxy.on('error', (err) => {
+              console.log('proxy error', err);
+            });
+            proxy.on('proxyReq', (proxyReq, req) => {
+              console.log(`Forwarding ${proxyReq.method} ${proxyReq.protocol}//${proxyReq.host}${proxyReq.path} to ${req.socket.remoteAddress}${req.url}`);
+            });
+            proxy.on('proxyRes', (proxyRes, req, res) => {
+              console.log(`Forwarding response ${req.socket.localAddress} to ${res.socket.remoteAddress}. Status=${proxyRes.statusCode}`);
+            });
+          },
+        },
+        '^/ws': {
+          target,
+          secure: false
+        }
+      },
+      port: 5173,
+      https: {
+        key: fs.readFileSync(keyFilePath),
+        cert: fs.readFileSync(certFilePath),
+      }
+    }
+  };
+});

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -0,0 +1,40 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="8.0.18" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="8.0.18" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ConfigurationProfile Include="appsettings.Development.json" />
+    <ConfigurationProfile Include="appsettings.Production.json" />
+    <ConfigurationProfile Include="appsettings.ProductionAws.json" />
+    <ConfigurationProfile Include="appsettings.Staging.json" />
+    <ConfigurationProfile Include="appsettings.json" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/CannedQueriesConfig.cs

@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQueriesConfig {
+   public List<CannedQuery>? Items { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/CannedQuery.cs

@@ -0,0 +1,6 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQuery {
+   public string? Name { get; set; }
+   public List<string>? Query { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/CannedQueriesController.cs

@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Microsoft.AspNetCore.Mvc;
+
+[ApiController]
+[Route("api/[controller]")]
+public class CannedQueriesController : ControllerBase {
+   readonly IConfiguration config;
+
+   public CannedQueriesController(IConfiguration config) =>
+      this.config = config;
+
+   [HttpGet(Name = "GetCannedQueries")]
+   public IEnumerable<CannedQuery>? Get() {
+      var cannedQueries = config.GetSection("CannedQueries").Get<List<CannedQuery>>()?.ToArray() ?? Array.Empty<CannedQuery>();
+      return cannedQueries;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/ConfigurationController.cs

@@ -0,0 +1,22 @@
+// using Bts.DataStreamWebApp.Server;
+
+// using Microsoft.AspNetCore.Mvc;
+
+// [ApiController]
+// [Route("api/[controller]")]
+// public class ConfigController : ControllerBase {
+//    private readonly IConfiguration _configuration;
+
+//    public ConfigController(IConfiguration configuration) {
+//       _configuration = configuration;
+//    }
+
+//    [HttpGet("websocket")]
+//    public ActionResult<DataStreamWebSocketConfig> GetWebSocketConfig() {
+//       var endpoint = $"{_configuration["Kestrel:Endpoints:Http:Url"]}/ws";
+//       var webSocketConfig = new DataStreamWebSocketConfig {
+//          Endpoint = endpoint // or Https based on your setup
+//       };
+//       return Ok(webSocketConfig);
+//    }
+// }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -0,0 +1,37 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Bts.DataStreamCore;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+   readonly IDataStreamFactory streamFactory;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+      var streamFactoryList = streamFactories.ToList();
+      streamFactory = streamFactoryList.Count == 1
+         ? streamFactoryList[0]
+         : new CompositeDataStreamFactory(streamFactoryList);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -0,0 +1,133 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System;
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamSubscription : IDisposable, IHostedService {
+   public readonly ViewStream ViewStream;
+   public DataPacket? LastFired;
+
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+
+   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
+      ViewStream = viewStream;
+      RequestId = requestId;
+      Id = id;
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public string Id { get; }
+   public JsonRpcId RequestId { get; }
+   public SubscriptionStatus Status { get; private set; }
+
+   public void Start() {
+      if (Status == SubscriptionStatus.Subscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Subscribed;
+      ViewStream.onChange += onChange;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Start();
+      return Task.CompletedTask;
+   }
+
+   public Task Clear() {
+      if (LastFired is null) {
+         return Task.CompletedTask;
+      }
+      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
+      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
+   }
+
+   public void Stop() {
+      if (Status == SubscriptionStatus.Unsubscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Unsubscribed;
+      ViewStream.onChange -= onChange;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Stop();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() =>
+      Stop();
+
+   async void onChange(object newValue) {
+      if (newValue is not DataPacket dp) {
+         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
+         return;
+      }
+      if (webSocket.State != WebSocketState.Open) {
+         logger.LogError(
+            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
+         Dispose();
+         return;
+      }
+      await sendDataPacket("subscribe", dp);
+   }
+
+   async Task sendDataPacket(string method, DataPacket dp) {
+      var serializerOpts = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      });
+      var res = JsonRpcRequest.CreateNotification(method, new DataStreamSubscriptionUpdate() {
+         SubscriptionId = Id,
+         Table = dp
+      });
+      var str = serialize(res, serializerOpts);
+      if (str is null) {
+         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
+         Dispose();
+         return;
+      }
+      logger.LogInformation(
+         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
+         dp.newRows.Count,
+         dp.schema.columnNames.Length,
+         str.Length);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      try {
+         await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+         LastFired = dp;
+      } catch (WebSocketException ex) {
+         logger.LogError($"sendDataPacket failed on webSocket.SendAsync: {ex}");
+      }
+   }
+
+   string? serialize<T>(T message, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      try {
+         return JsonSerializer.Serialize(message, opts);
+      } catch (Exception ex) {
+         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
+      }
+      return null;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscriptionUpdate.cs

@@ -0,0 +1,13 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using BCG;
+
+using System.Text.Json.Serialization;
+
+public class DataStreamSubscriptionUpdate {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("table")]
+   public DataPacket? Table { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerConfig.cs

@@ -0,0 +1,89 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Bts.PubSub;
+
+using CommandLine;
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamWebAppServerConfig {
+   /// <summary>
+   /// The login user for sparkd
+   /// </summary>
+   [JsonPropertyName("user")]
+   [Option("user", HelpText = "The sparkd login user")]
+   public string? User { get; set; }
+
+   /// <summary>
+   /// The login password for sparkd
+   /// </summary>
+   [JsonPropertyName("password")]
+   [Option("pass", HelpText = "The sparkd login password")]
+   public string? Password { get; set; }
+
+   /// <summary>
+   /// The config server url for sparkd
+   /// </summary>
+   [JsonPropertyName("sparkd")]
+   [Option("sparkd", HelpText = "The sparkd config server")]
+   public string? Sparkd { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+   [Option("content-root-path")]
+   [JsonPropertyName("content_root_path")]
+   public string? ContentRootPath { get; set; }
+
+   [Option("web-root-path")]
+   [JsonPropertyName("web_root_path")]
+   public string? WebRootPath { get; set; }
+
+   [Option("environment-name")]
+   [JsonPropertyName("environment_name")]
+   public string? EnvironmentName { get; set; }
+
+   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
+
+   [Option(
+      "config",
+      MetaValue = "PATH",
+      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
+   [JsonIgnore]
+   public string? ConfigFile { get; set; }
+
+   public static DataStreamWebAppServerConfig Default() =>
+      new() {
+         User = "bts",
+         PubSub = PubSubConfig.Default(),
+      };
+
+   public void MergeWith(DataStreamWebAppServerConfig other) {
+      User ??= other.User;
+      Password ??= other.Password;
+      PubSub ??= other.PubSub;
+      Sparkd ??= other.Sparkd;
+      ContentRootPath ??= other.ContentRootPath;
+      WebRootPath ??= other.WebRootPath;
+      ConfigFile ??= other.ConfigFile;
+   }
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      if (ConfigFile is null) {
+         return;
+      }
+      var cfgString = File.ReadAllText(ConfigFile);
+      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
+         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      MergeWith(parsedCfg);
+      ConfigFile = null;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketConfig.cs

@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class DataStreamWebSocketConfig {
+   public string? Endpoint { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -0,0 +1,151 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+   readonly IDataStreamFactory streamFactory;
+
+   public DataStreamWebSocketServer(WebSocket webSocket, IDataStreamFactory streamFactory, ILogger logger) {
+      this.webSocket = webSocket;
+      this.streamFactory = streamFactory;
+      this.logger = logger;
+   }
+
+   public async Task<int> Run(CancellationToken cancellationToken) {
+      logger.LogInformation("DataStreamWebSocketServer running");
+      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
+         await (req.Method switch {
+            "subscribe" => onSubscribe(req, cancellationToken),
+            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
+            _ => Task.CompletedTask,
+         });
+      }
+      logger.LogInformation("DataStreamWebSocketServer stopped");
+      return 0;
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
+   /// change(s), use <see cref="DisposeAsync"/>.
+   /// </summary>
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      disposeSubscriptions();
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
+   /// notifications to the client (unlike <see cref="Dispose"/>).
+   /// </summary>
+   public async ValueTask DisposeAsync() {
+      GC.SuppressFinalize(this);
+      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
+      disposeSubscriptions();
+      // Send a 'subscription status change' msg to the client for each one.
+      foreach (var (requestId, id) in finalSubs) {
+         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
+      }
+   }
+
+   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      if (!SubscribeRequest.Parse(req, out var subReq, out var parseErr, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
+      disposeSubscription(subId);
+      if (!subReq.ParseQuery(streamFactory, out var vs, out parseErr, out errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      if (subscriptions.TryGetValue(subId, out var oldSub)) {
+         logger.LogInformation("Disposing existing subscription with id {SubscriptionId}", subId);
+         oldSub.Dispose();
+      }
+      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
+      subscriptions[subId] = sub;
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
+      vs.ErrorChanged += OnStreamError;
+      await sub.StartAsync(cancellationToken);
+
+      async void OnStreamError(Exception exception) {
+         logger.LogError("Error in ViewStream (disposing subscription): {Exception}", exception);
+         subscriptions.Remove(sub.Id);
+         sub.Dispose();
+         var error = new SubscriptionError() {
+            ErrorMessage = "Error in ViewStream (disposing subscription)"
+         };
+         if (exception is not null) {
+            error.ErrorMessage += ". " + exception.Message;
+            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
+         }
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)ErrorCode.ViewStreamError, Message = "Error in ViewStream (disposing subscription)", Data = error });
+         await send(rpcError, cancellationToken);
+      }
+   }
+
+   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      var subId = req.Params?.SubscriptionId ?? string.Empty;
+      if (!disposeSubscription(subId)) {
+         logger.LogInformation("Unable to dispose subscription. Subscription with id {Id} not found.", subId);
+      }
+      await sendSubscriptionStatus(req.Id.Value, subId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   bool disposeSubscription(string subId) {
+      if (!subscriptions.TryGetValue(subId, out var sub)) {
+         return false;
+      }
+      logger.LogInformation("Disposing subscription with id {Id}", subId);
+      sub.Dispose();
+      subscriptions.Remove(subId);
+      return true;
+   }
+
+   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
+      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
+      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
+         SubscriptionId = subId,
+         Status = status,
+      });
+      return send(response, cancellationToken);
+   }
+
+   Task send<T>(T message, CancellationToken cancellationToken) {
+      if (message is null) {
+         return Task.CompletedTask;
+      }
+      var str = JsonSerializer.Serialize(message);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
+      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
+   }
+
+   void disposeSubscriptions() {
+      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
+      subscriptions.Values.DisposeAll();
+      subscriptions.Clear();
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile

@@ -0,0 +1,11 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
+ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
+
+RUN dotnet dev-certs https --trust
+COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
+WORKDIR "/home/bts/bin"

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/CannedQueriesResponse.cs

@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQueriesResponse {
+   public List<CannedQuery>? Items { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/ErrorCode.cs

@@ -0,0 +1,7 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+enum ErrorCode {
+   None = 0,
+   InvalidSubscriptionRequest,
+   ViewStreamError,
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -0,0 +1,78 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class SubscribeRequest {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   internal static bool Parse(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)]out SubscribeRequest? subscription, [NotNullWhen(false)]out SubscriptionError? error, out ErrorCode errorCode) {
+      subscription = null;
+      errorCode = ErrorCode.None;
+      var paramsJsonObj = req.Params?.AsObject();
+      error = null;
+      if (paramsJsonObj is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "'params' object not found" };
+         return false;
+      }
+      subscription = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subscription is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "'params' object invalid" };
+         return false;
+      }
+      return true;
+   }
+
+   internal bool ParseQuery(
+      IDataStreamFactory streamFactory,
+      [NotNullWhen(true)] out ViewStream? stream,
+      [NotNullWhen(false)] out SubscriptionError? error,
+      out ErrorCode errorCode
+   ) {
+      stream = null;
+      errorCode = ErrorCode.None;
+      error = null;
+      if (Query is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "'params.query' field not found (expected a string)" };
+         return false;
+      }
+      var exp = DataStreamExpressionsInterpreter.Evaluate(Query);
+      if (exp is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "'params.query' expression has invalid syntax" };
+         return false;
+      }
+      Exception? exception = null;
+      try {
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+      } catch (Exception ex) {
+         exception = ex;
+      }
+      if (stream is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)" };
+         if (exception is not null) {
+            error.ErrorMessage += ". " + exception.Message;
+            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
+         }
+         return false;
+      }
+      return true;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscriptionError.cs

@@ -0,0 +1,11 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class SubscriptionError {
+   [JsonPropertyName("error_message")]
+   public string? ErrorMessage { get; set; }
+
+   [JsonPropertyName("stack_trace")]
+   public List<string>? StackTrace { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscriptionStatusMessage.cs

@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+enum SubscriptionStatus {
+   None,
+   Subscribed,
+   Unsubscribed,
+}
+
+class SubscriptionStatusMessage {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("status")]
+   [JsonConverter(typeof(JsonStringEnumConverter))]
+   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/UnsubscribeRequest.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class UnsubscribeRequest {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -0,0 +1,87 @@
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+using Bts.SharedStreams.Services;
+
+using Spark.Api.Services;
+using Spark.Core.Services;
+
+using System.Reactive.Linq;
+using System.Reflection;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration, builder.Environment.EnvironmentName);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser("/browse");
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder, string environment) {
+   // We prefix appsettings with the exe name because multiple exes will usually be in the build dir
+   // (this way, each exe can still load its own appsettings files from the current directory).
+   var exeName = Assembly.GetCallingAssembly().GetName().Name;
+   AddJsonFile($"{exeName}.appsettings.json");
+   AddJsonFile($"{exeName}.appsettings.{environment}.json");
+
+   var cfg = configBuilder.GetSection("config")?.Value;
+   if (!SetBasePath(cfg)) {
+      AddJsonFile(cfg);
+   }
+
+   bool SetBasePath(string? path) {
+      if (!Directory.Exists(path)) {
+         return false;
+      }
+      configBuilder.SetBasePath(path);
+      return true;
+   }
+
+   bool AddJsonFile(string? path) {
+      if (!File.Exists(path)) {
+         return false;
+      }
+      configBuilder.AddJsonFile(path);
+      return true;
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser()
+      .AddControllers().Services
+      .AddEndpointsApiExplorer()
+      .AddSwaggerGen();
+   services.AddSpaStaticFiles();
+   services.AddSpark(config.GetSection("Spark"))
+      .AddSparkApi(config.GetSection("SparkApi"))
+      .AddMasterWorker(config.GetSection("MasterWorker"))
+      .AddViewStreamDatabase()
+      .AddPrimitiveStreams(config.GetSection("PrimitiveStreams"))
+      .AddMarketDataStreams()
+      .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
+      .AddCompositeDataStreamFactory()
+      .AddPubSubClient(config.GetSection("PubSub"));
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:8080",
+      "environmentVariables": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:8080;https://localhost:8443",
+      "environmentVariables": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -0,0 +1,52 @@
+{
+  "PubSub": {
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    }
+  },
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ]
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.NaturalLanguageCustom.json

@@ -0,0 +1,45 @@
+{
+  "NaturalLanguagePrompt": {
+    "prompts": [
+      {
+        "category": "Impact of financial events on the market",
+        "description": "Show how trading metrics (e.g., price movement, volume) are affected by events related to finance (rate announcements, earnings reports, etc.).",
+        "prompt": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+        "examples": [
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(which: \"last 5\", duration: \"+5min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "How do OPEC meetings affect the price of Crude futures? Examine the 10 minutes after each meeting this year.",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(duration: \"+10min\", predicate: \"timestamp.year==2024\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Show me how the SP 500 price moves after FOMC rate announcements that day",
+            "dsl": [
+              "time_idx=EventTimeIndex.FomcRateAnnouncements(which: \"timestamp.year==2024\", duration: \"+10min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ES\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Analyze the impact of the FED's interest rate hikes in 2022 on the 10-year Treasury yield, focusing on the 2-week period following each hike.",
+            "dsl": [
+              "time_idx=FedInterestRateAnnouncements(predicate: \"timestamp.year==2022 && change_pct>0\", duration: \"+2 weeks\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ZN\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx)"
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Production.json

@@ -1,5 +1,8 @@
 {
   "Spark": {
     "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true
   }
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.ProductionAws.json

@@ -0,0 +1,115 @@
+{
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "Spark": {
+    "ConfigServer": "http://172.30.2.181:8000/config"
+  },
+  "SparkApi": {
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
+  },
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://172.30.2.181:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://172.30.2.181:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  },
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://172.30.2.181:8080"
+      },
+      "Https": {
+        "Url": "https://172.30.2.181:8443"
+      }
+    }
+  },
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ]
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Staging.json

@@ -0,0 +1,44 @@
+{
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ],
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -0,0 +1,443 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*",
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://0.0.0.0:8080"
+      },
+      "HttpIpv6": {
+        "Url": "http://[::1]:8080"
+      },
+      "Https": {
+        "Url": "https://0.0.0.0:8443"
+      },
+      "HttpsIpv6": {
+        "Url": "https://[::1]:8443"
+      }
+    }
+  },
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "PrimitiveStream": {
+    "Assemblies": [
+      "DataStream.dll",
+      "SharedStreams.dll"
+    ],
+    "ExcludedTypes": [
+      "SettlementPriceStream",
+      "OHLCVStream"
+    ]
+  },
+  "StreamDatabase": {
+    "Path": "/home/bts/.local/share/bts/silver_pine/store.db",
+    "Query": "select * from metadata"
+  },
+  "NaturalLanguageStream": {
+    "prompts": [
+      {
+        "category": "Settlement",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). ",
+          "The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). ",
+          "The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement ",
+          "(symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have ",
+          "datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know ",
+          "what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however ",
+          "you should use the Today's real date"
+        ],
+        "prompt": "What was the settlement for SP 500 Future on June 8th?",
+        "examples": [
+          {
+            "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
+            "dsl": "Settlement(\"ES\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.",
+            "dsl": "Settlement(\"SIZ4\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.",
+            "dsl": "Settlement(\"CL\", 10/10/2024)"
+          },
+          {
+            "natural_language": "Find today's settlement of the 10 year treasury future.",
+            "dsl": "Settlement(\"ZN\", 10/11/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NGZ4\", 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.",
+            "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25  spread for the first two weeks of September.",
+            "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NG\", 9/6/2024)"
+          }
+        ]
+      },
+      {
+        "category": "BBO",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "prompt": "Show me bbo for SP 500 futures on Sep 23rd of this year.",
+        "examples": [
+          {
+            "natural_language": "Get SP500 future bbo from Wednesday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", exclusive_end: true)"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA L1 books from Oct 3",
+            "dsl": "Bbo(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\")"
+          },
+          {
+            "natural_language": "Show me L1 data for Crude oil futures this week",
+            "dsl": "Bbo(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Fetch hourly bbo for Gold futures this week",
+            "dsl": "Bbo(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute bbo for Tesla yesterday?",
+            "dsl": "Bbo(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "Order flow imbalance",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is OrderImbalance(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "levels (the number of price levels to examine)",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "inclusive_end (a bool flag that, when true, makes the end time inclusive). ",
+          "Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "examples": [
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", \"PT1M\")"
+          },
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday for the top 5 price levels.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", levels: 5, interval: \"PT1M\")"
+          },
+          {
+            "natural_language": "Get orderflow imbalance for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA order flow from Oct 3",
+            "dsl": "OrderImbalance(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Show me order imbalance for Crude oil futures this week",
+            "dsl": "OrderImbalance(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "Fetch hourly orderflow for Gold futures this week",
+            "dsl": "OrderImbalance(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute orderflow for Tesla yesterday?",
+            "dsl": "OrderImbalance(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "VWAP",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
+        "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
+        "examples": [
+          {
+            "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
+            "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
+            "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
+            "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
+          },
+          {
+            "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
+            "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
+          }
+        ]
+      },
+      {
+        "category": "Price Diff",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
+        "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
+        "examples": [
+          {
+            "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
+            "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
+            "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
+            "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Absolute Volume",
+        "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
+        "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
+        "examples": [
+          {
+            "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
+            "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
+            "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
+          },
+          {
+            "natural_language": "Which 10 year Spread traded the highest volume last week?",
+            "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
+          },
+          {
+            "natural_language": "Which Corn Spread traded the most today?",
+            "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Which Gold Future traded the most today?",
+            "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Roll",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ",
+        "prompt": "When does the roll typically occur In  SP 500?",
+        "examples": [
+          {
+            "natural_language": "When does the roll typically occur In CL?",
+            "dsl": "FuturesRoll(\"CL\")"
+          },
+          {
+            "natural_language": "When does the roll typically occur for 10 year treasury?",
+            "dsl": "FuturesRoll(\"ZN\")"
+          },
+          {
+            "natural_language": "When does the roll usually occur for Gold Future",
+            "dsl": "FuturesRoll(\"GC\")"
+          },
+          {
+            "natural_language": "At what date does the future rollover happens for SP500?",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "When does the future rollover happens for the Corn Future?",
+            "dsl": "FuturesRoll(\"ZC\")"
+          }
+        ]
+      },
+      {
+        "category": "Daily Move",
+        "description": "This is a few shot learning example for an NL to DSL. ",
+        "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?",
+        "examples": [
+          {
+            "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)"
+          },
+          {
+            "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)"
+          },
+          {
+            "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open",
+            "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          },
+          {
+            "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          }
+        ]
+      },
+      {
+        "category": "Average Volume",
+        "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
+        "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
+        "examples": [
+          {
+            "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "Usually when does the open interest move from one contract to another in WTI?",
+            "dsl": "FuturesRoll(\"CL\")"
+          }
+        ]
+      },
+      {
+        "category": "Events",
+        "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.",
+        "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?",
+        "examples": [
+          {
+            "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?",
+            "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)"
+          },
+          {
+            "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?",
+            "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)"
+          }
+        ]
+      },
+      {
+        "category": "Indicator, Standard Deviation",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when ",
+          "the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, ",
+          "where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and ",
+          "amounts of the last 4+ standard deviation moves in WTI futures\" would be: ",
+          "r=standardDeviation(\"ES\", \"90d\")\n",
+          "r=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\n",
+          "r=filter(r,\"abs_daily_change >= comp\"). ",
+          "The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, ",
+          "and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, ",
+          "abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters ",
+          "(like a sql where) of rows where abs_daily_change >= comp"
+        ],
+        "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP  futures ?",
+        "examples": [
+          {
+            "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?",
+            "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation",
+            "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation",
+            "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation",
+            "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          }
+        ]
+      }
+    ]
+  },
+  "SparkApi": {
+    "IsEnabled": false,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  },
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://pubsub:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://pubsub:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf

@@ -0,0 +1,15 @@
+Logging {
+  value {
+    console {
+      default_level: "Info"
+    }
+    syslog {
+      default_level: "Info"
+    }
+    file {
+      default_level: "Info"
+    }
+    initial_queue_size: 120001
+    max_queue_growth: 4
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.7" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs

@@ -0,0 +1,59 @@
+namespace Bts.Extensions.Hosting;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+
+using System.Diagnostics;
+
+public static class HostBuilderExtensions {
+   public static IHostBuilder CreateBtsBuilder(string[] args) {
+      // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
+      // because want the name of the assembly that called this function, not the assembly that will eventually invoke
+      // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
+      // Get the name of the assembly that called this function. That will act as the "appsettings"
+      // json file prefix
+      var exeName = GetCallingAssemblyName();
+      return Host.CreateDefaultBuilder(args)
+         .ConfigureAppConfiguration((context, configBuilder) =>
+            ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
+   }
+
+   public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
+      var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
+      AddJsonFile($"{baseFileName}.json");
+      if (environment is not null) {
+         AddJsonFile($"{baseFileName}.{environment}.json");
+      }
+
+      var cfg = configBuilder.Build().GetSection("config")?.Value;
+      if (!SetBasePath(cfg)) {
+         AddJsonFile(cfg);
+      }
+
+      bool SetBasePath(string? path) {
+         if (!Directory.Exists(path)) {
+            return false;
+         }
+         configBuilder.SetBasePath(path);
+         return true;
+      }
+
+      bool AddJsonFile(string? path) {
+         if (!File.Exists(path)) {
+            return false;
+         }
+         configBuilder.AddJsonFile(path);
+         return true;
+      }
+   }
+
+   public static string GetCallingAssemblyName() =>
+      new StackTrace()
+         .GetFrame(2)
+         ?.GetMethod()
+         ?.Module
+         ?.Assembly
+         ?.GetName()
+         ?.Name
+         ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
+}